From 65a542579b4849a6b064c8c8b3baf16e31119294 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 22 Nov 2016 11:14:11 +0100 Subject: updated README.md --- lib/README.md | 2 +- lib/dll/example/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/README.md b/lib/README.md index 18793d4..b40442c 100644 --- a/lib/README.md +++ b/lib/README.md @@ -47,7 +47,7 @@ The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.lib". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md index 4908663..223e473 100644 --- a/lib/dll/example/README.md +++ b/lib/dll/example/README.md @@ -39,7 +39,7 @@ The header files from `include\` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.dll". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll ``` -- cgit v0.12 From de0cf5de640eb56775f44c0f4c7c7a065e00447c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 22 Nov 2016 11:20:02 +0100 Subject: fixed LZ4IO_compressMultipleFilenames --- programs/lz4io.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index bc2ba95..5e45e72 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -593,10 +593,12 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const size_t suffixSize = strlen(suffix); cRess_t const ress = LZ4IO_createCResources(); + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + /* loop on each file */ for (i=0; i Date: Tue, 22 Nov 2016 11:18:07 -0800 Subject: new test case with fPIE --- .travis.yml | 2 +- lib/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c127fa5..e3f476b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -130,7 +130,7 @@ matrix: packages: - clang - - env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror' COMPILER=gcc-4.4 + - env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror && make clean && CFLAGS=-fPIC LDFLAGS="-pie -fPIE -D_FORTIFY_SOURCE=2" make -C programs' COMPILER=gcc-4.4 dist: trusty sudo: required addons: diff --git a/lib/Makefile b/lib/Makefile index 28242cc..a9c7df8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -85,7 +85,7 @@ lib-release: lib liblz4.a: *.c ifeq ($(BUILD_STATIC),yes) @echo compiling static library - @$(CC) $(FLAGS) -c $^ + @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ @$(AR) rcs $@ *.o endif -- cgit v0.12 From edc132b69f0f653029e79ecdbf0a5335ea4ed0ea Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 22 Nov 2016 11:37:51 -0800 Subject: lz4 release build compatible with PIE --- programs/Makefile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 9939a44..caef394 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -34,12 +34,12 @@ MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib VOID := /dev/null -CFLAGS ?= -O3 # allows custom optimization flags. For example : CFLAGS="-O2 -g" make -CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ +CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ +CFLAGS ?= -O3 +DEBUGFLAGS=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ - -Wpointer-arith -Wstrict-aliasing=1 + -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) @@ -54,17 +54,18 @@ endif default: lz4-release +lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o + $(CC) $(FLAGS) $^ -o lz4$(EXT) + all: lz4 lz4c all32: CFLAGS+=-m32 all32: all -lz4: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o $@$(EXT) - -lz4-release: CFLAGS := -O3 -lz4-release: lz4 +lz4: CFLAGS += $(DEBUGFLAGS) +lz4: lz4-release +lz4c: CFLAGS += $(DEBUGFLAGS) lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -- cgit v0.12 From f4c74e6fac82d090d4511dc2d83b09725ff5e639 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 22 Nov 2016 11:52:43 -0800 Subject: library release build compatible with environment variable --- lib/Makefile | 41 +++++++++++++++++++++-------------------- programs/Makefile | 2 +- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index a9c7df8..2b47fb0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -38,22 +38,23 @@ LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCR LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) -LIBVER := $(shell echo $(LIBVER_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) BUILD_STATIC:= yes -DESTDIR?= -PREFIX ?= /usr/local -CPPFLAGS= -DXXH_NAMESPACE=LZ4_ -CFLAGS ?= -O3 -CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wpointer-arith -Wstrict-aliasing=1 +DESTDIR ?= +PREFIX ?= /usr/local +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include + +CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ +CFLAGS ?= -O3 +DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ + -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -LIBDIR?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include # OS X linker doesn't support -soname, and use different extension @@ -72,16 +73,16 @@ endif default: lib-release +lib-release: liblz4.a liblz4 + +lib: CFLAGS += $(DEBUGFLAGS) +lib: lib-release + all: lib all32: CFLAGS+=-m32 all32: all -lib: liblz4.a liblz4 - -lib-release: CFLAGS := -O3 -lib-release: lib - liblz4.a: *.c ifeq ($(BUILD_STATIC),yes) @echo compiling static library @@ -114,10 +115,10 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) liblz4.pc: liblz4.pc.in Makefile @echo creating pkgconfig @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@VERSION@|$(LIBVER)|' \ - $< >$@ + -e 's|@LIBDIR@|$(LIBDIR)|' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@VERSION@|$(LIBVER)|' \ + $< >$@ install: lib liblz4.pc @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ diff --git a/programs/Makefile b/programs/Makefile index caef394..504416f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -36,7 +36,7 @@ VOID := /dev/null CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 -DEBUGFLAGS=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -- cgit v0.12 From 605512a7b44ed5547fa02cf2282a562fcd0d9935 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 22 Nov 2016 12:31:04 -0800 Subject: updated NEWS --- NEWS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index bb9d1fb..63680a8 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,10 @@ +v1.7.5 +fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) + v1.7.4 Improved : much better speed in -mx32 mode cli : fix : Large file support in 32-bits mode on Mac OS-X -fix : compilation on gcc 4.4 (#282), reported by Antoine Martin +fix : compilation on gcc 4.4 (#272), reported by Antoine Martin v1.7.3 Changed : moved to versioning; package, cli and library have same version number -- cgit v0.12 From 018ddf799917ee5c68b5266d6f42277fa6750080 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 22 Nov 2016 12:34:17 -0800 Subject: numbering --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 63680a8..f6874ce 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -v1.7.5 +v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) v1.7.4 -- cgit v0.12 From 544290d587cd91261542d928f7e36192cf5e5df5 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Sun, 27 Nov 2016 23:40:53 +0900 Subject: Add "man" target to Makefile - `make man` makes manpage lz4.1 from lz4.1.md - It needs `gem` and `ronn` which is same as xxhash.1.md is required. --- programs/Makefile | 27 +++++++ programs/lz4.1.md | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 programs/lz4.1.md diff --git a/programs/Makefile b/programs/Makefile index 504416f..ffbcbf1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,6 +27,17 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## +# Version numbers +LIBVER_SRC := ../lib/lz4.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + DESTDIR ?= PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin @@ -42,6 +53,10 @@ DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) +LZ4_VERSION=$(LIBVER) +MD2ROFF =ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" + # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -80,6 +95,18 @@ clean: @echo Cleaning completed +lz4.1: lz4.1.md + cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: lz4.1 + +clean-man: + rm lz4.1 + +preview-man: clean-man man + man ./lz4.1 + + #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD, Hurd and #FreeBSD targets diff --git a/programs/lz4.1.md b/programs/lz4.1.md new file mode 100644 index 0000000..64ce428 --- /dev/null +++ b/programs/lz4.1.md @@ -0,0 +1,206 @@ +lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files +================================================================ + +SYNOPSIS +-------- + +`lz4` [*OPTIONS*] [-|INPUT-FILE] + +`unlz4` is equivalent to `lz4 -d` + +`lz4cat` is equivalent to `lz4 -dcfm` + +When writing scripts that need to decompress files, +it is recommended to always use the name `lz4` with appropriate arguments +(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. + + +DESCRIPTION +----------- + +`lz4` is an extremely fast lossless compression algorithm, +based on **byte-aligned LZ77** family of compression scheme. +`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with +multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the `.lz4` format. + +### Difference between lz4 and gzip + +`lz4` supports a command line syntax similar +_but not identical_ to `gzip(1)`. +Differences are : + + * `lz4` preserves original files + * `lz4` compresses a single file by default (use `-m` for multiple files) + * `lz4 file1 file2` means : compress file1 _into_ file2 + * When no destination name is provided, compressed file name receives + a `.lz4` suffix + * When no destination name is provided, if `stdout` is _not_ the console, + it becomes the output (like a silent `-c`). + Therefore `lz4 file > /dev/null` will not create `file.lz4` + * `lz4 file` shows real-time statistics during compression + (use `-q` to silent them) + +Default behaviors can be modified by opt-in commands, described below. +`lz4 --quiet --multiple` more closely mimics `gzip` behavior. + +### Concatenation of .lz4 files + +It is possible to concatenate `.lz4` files as is. +`lz4` will decompress such files as if they were a single `.lz4` file. +For example: + lz4 file1 > foo.lz4 + lz4 file2 >> foo.lz4 + +then + lz4cat foo.lz4 + +is equivalent to : + cat file1 file2 + + +OPTIONS +------- + +### Short commands concatenation + +In some cases, some options can be expressed using short command `-x` +or long command `--long-word`. +Short commands can be concatenated together. +For example, `-d -c` is equivalent to `-dc`. +Long commands cannot be concatenated. +They must be clearly separated by a space. + +### Multiple commands + +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. + +### Operation mode + +* `-z` `--compress`: + Compress. + This is the default operation mode when no operation mode option is + specified, no other operation mode is implied from the command name + (for example, `unlz4` implies `--decompress`), + nor from the input file name + (for example, a file extension `.lz4` implies `--decompress` by default). + `-z` can also be used to force compression of an already compressed + `.lz4` file. + +* `-d` `--decompress` `--uncompress`: + Decompress. + `--decompress` is also the default operation when the input filename has an + `.lz4` extension. + +* `-t` `--test`: + Test the integrity of compressed `.lz4` files. + The decompressed data is discarded. + No files are created nor removed. + +* `-b#`: + Benchmark mode, using `#` compression level. + +### Operation modifiers + +* `-#`: + Compression level, with # being any value from 1 to 16. + Higher values trade compression speed for compression ratio. + Values above 16 are considered the same as 16. + Recommended values are 1 for fast compression (default), + and 9 for high compression. + Speed/compression trade-off will vary depending on data to compress. + Decompression speed remains fast at all settings. + +* `-f` `--[no-]force`: + This option has several effects: + + If the target file already exists, overwrite it without prompting. + + When used with `--decompress` and `lz4` cannot recognize the type of + the source file, copy the source file as is to standard output. + This allows `lz4cat --force` to be used like `cat (1)` for files + that have not been compressed with `lz4`. + +* `-c` `--stdout` `--to-stdout`: + Force write to standard output, even if it is the console. + +* `-m` `--multiple`: + Multiple file names. + By default, the second filename is used as the destination filename + for the compressed file. + With `-m`, it is possible to specify any number of input filenames. + Each of them will be compressed independently, and the resulting name of + each compressed file will be `filename.lz4`. + +* `-B#`: + Block size \[4-7\](default : 7)
+ `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB + +* `-BD`: + Block dependency (improves compression ratio on small blocks) + +* `--[no-]frame-crc`: + Select frame checksum (default:enabled) + +* `--[no-]content-size`: + Header includes original size (default:not present)
+ Note : this option can only be activated when the original size can be + determined, hence for a file. It won't work with unknown source size, + such as stdin or pipe. + +* `--[no-]sparse`: + Sparse mode support (default:enabled on file, disabled on stdout) + +* `-l`: + Use Legacy format (typically used for Linux Kernel compression)
+ Note : `-l` is not compatible with `-m` (`--multiple`) + +### Other options + +* `-v` `--verbose`: + Verbose mode + +* `-q` `--quiet`: + Suppress warnings and real-time statistics; specify twice to suppress + errors too + +* `-h` `-H` `--help`: + Display help/long help and exit + +* `-V` `--version`: + Display Version number and exit + +* `-k` `--keep`: + Don't delete source file. + This is default behavior anyway, so this option is just for compatibility + with `gzip(1)` / `xz(1)`. + + +### Benchmark mode + +* `-b#`: + Benchmark file(s), using # compression level + +* `-e#`: + Benchmark multiple compression levels, from b# to e# (included) + +* `-i#`: + Minimum evaluation in seconds \[1-9\] (default : 3) + +* `-r`: + Operate recursively on directories + + +BUGS +---- + +Report bugs at: https://github.com/lz4/lz4/issues + + +AUTHOR +------ + +Yann Collet -- cgit v0.12 From 009f06e01d424861048727ab5d9486f7a60a3a6e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 27 Nov 2016 23:23:40 -0800 Subject: minor man page update --- programs/lz4.1 | 389 ++++++++++++++++++++++-------------------------------- programs/lz4.1.md | 10 +- 2 files changed, 168 insertions(+), 231 deletions(-) diff --git a/programs/lz4.1 b/programs/lz4.1 index 2c94033..d73a7ed 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,251 +1,184 @@ -\." -\." lz4.1: This is a manual page for 'lz4' program. This file is part of the -\." lz4 project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lz4 "1" "2015-03-21" "lz4" "User Commands" -.SH NAME -\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] -.PP -.B unlz4 -is equivalent to -.BR "lz4 \-d" -.br -.B lz4cat -is equivalent to -.BR "lz4 \-dcfm" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lz4 -with appropriate arguments -.RB ( "lz4 \-d" -or -.BR "lz4 \-dc" ) -instead of the names -.B unlz4 -and -.BR lz4cat . - - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .lz4 -format. - -.B lz4 -supports a command line syntax similar \fIbut not identical\fR to -.BR gzip (1). -Differences are : - \fBlz4\fR preserves original files - \fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files) - \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 - When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix - When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR) - Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR - \fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them) - -Default behaviors can be modified by opt-in commands, described below. -\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .lz4 files" -It is possible to concatenate -.B .lz4 -files as is. -.B lz4 -will decompress such files as if they were a single -.B .lz4 -file. For example: - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - then - lz4cat foo.lz4 - is equivalent to : - cat file1 file2 - -.PP - -.SH OPTIONS +. +.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands" +. +.SH "NAME" +\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files +. +.SH "SYNOPSIS" +\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR +. +.P +\fBunlz4\fR is equivalent to \fBlz4 \-d\fR +. +.P +\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR +. +.P +When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. +. +.SH "DESCRIPTION" +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +. +.SS "Difference between lz4 and gzip" +\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : +. +.IP "\(bu" 4 +\fBlz4\fR preserves original files +. +.IP "\(bu" 4 +\fBlz4\fR compresses a single file by default (use \fB\-m\fR for multiple files) +. +.IP "\(bu" 4 +\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 +. +.IP "\(bu" 4 +When no destination name is provided, compressed file name receives a \fB\.lz4\fR suffix +. +.IP "\(bu" 4 +When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB\-c\fR)\. Therefore \fBlz4 file > /dev/null\fR will not create \fBfile\.lz4\fR +. +.IP "\(bu" 4 +\fBlz4 file\fR shows real\-time statistics during compression (use \fB\-q\fR to silent them) +. +.IP "" 0 +. +.P +Default behaviors can be modified by opt\-in commands, described below\. \fBlz4 \-m\fR more closely follows \fBgzip\fR command line syntax, supporting compression of multiple files into \fB\.lz4\fR, the last remaining difference being that source files are preserved by default\. Since it\'s also possible to erase source file using opt\-in command \fB\-\-rm\fR, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.SS "Concatenation of \.lz4 files" +It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 +. +.P +then lz4cat foo\.lz4 +. +.P +is equivalent to : cat file1 file2 +. +.SH "OPTIONS" . .SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. +In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. +. .SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. +When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. . .SS "Operation mode" +. +.TP +\fB\-z\fR \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlz4 -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .lz4 -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .lz4 -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .lz4 -extension. -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .lz4 -files. -The decompressed data is discarded. -No files are created nor removed. - -.TP -.BR \-b# -Benchmark mode, using # compression level. - +\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR +Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. +. +.TP +\fB\-t\fR \fB\-\-test\fR +Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. +. +.TP +\fB\-b#\fR +Benchmark mode, using \fB#\fR compression level\. . .SS "Operation modifiers" +. +.TP +\fB\-#\fR +Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. +. +.TP +\fB\-f\fR \fB\-\-[no\-]force\fR +This option has several effects: +. +.IP +If the target file already exists, overwrite it without prompting\. +. +.IP +When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. +. +.TP +\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR +Force write to standard output, even if it is the console\. +. +.TP +\fB\-m\fR \fB\-\-multiple\fR +Multiple file names\. By default, the second filename is used as the destination filename for the compressed file\. With \fB\-m\fR, it is possible to specify any number of input filenames\. Each of them will be compressed independently, and the resulting name of each compressed file will be \fBfilename\.lz4\fR\. This mode also reduces notification level\. +. +.TP +\fB\-B#\fR +Block size [4\-7](default : 7) +. +.br +\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB +. +.TP +\fB\-BD\fR +Block dependency (improves compression ratio on small blocks) +. +.TP +\fB\-\-[no\-]frame\-crc\fR +Select frame checksum (default:enabled) +. +.TP +\fB\-\-[no\-]content\-size\fR +Header includes original size (default:not present) +. +.br +Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. +. .TP -.B \-# - compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - -.TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lz4 -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lz4cat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lz4 . -.RE - -.TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - -.TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, it is possible to specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.lz4 -. - -.TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB -.TP -.B \-BD - block dependency (improves compression ratio on small blocks) -.TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) -.TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. -.TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) -.TP -.B \-l - use Legacy format (typically used for Linux Kernel compression) - note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR) +\fB\-\-[no\-]sparse\fR +Sparse mode support (default:enabled on file, disabled on stdout) +. +.TP +\fB\-l\fR +Use Legacy format (typically used for Linux Kernel compression) +. +.br +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) . .SS "Other options" +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-v\fR \fB\-\-verbose\fR +Verbose mode +. .TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too +\fB\-q\fR \fB\-\-quiet\fR +Suppress warnings and real\-time statistics; specify twice to suppress errors too +. .TP -.B \-h/\-H ", " --help - display help/long help and exit +\fB\-h\fR \fB\-H\fR \fB\-\-help\fR +Display help/long help and exit +. .TP -.BR \-V ", " \--version - display Version number and exit +\fB\-V\fR \fB\-\-version\fR +Display Version number and exit +. .TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. - +\fB\-k\fR \fB\-\-keep\fR +Don\'t delete source file\. This is default behavior anyway, so this option is just for compatibility with \fBgzip(1)\fR / \fBxz(1)\fR\. . .SS "Benchmark mode" +. .TP -.B \-b# - benchmark file(s), using # compression level +\fB\-b#\fR +Benchmark file(s), using # compression level +. .TP -.B \-e# - benchmark multiple compression levels, from b# to e# (included) +\fB\-e#\fR +Benchmark multiple compression levels, from b# to e# (included) +. .TP -.B \-i# - minimum evaluation in seconds [1-9] (default : 3) +\fB\-i#\fR +Minimum evaluation in seconds [1\-9] (default : 3) +. .TP -.B \-r - operate recursively on directories - - -.SH BUGS -Report bugs at: https://github.com/Cyan4973/lz4/issues - -.SH AUTHOR +\fB\-r\fR +Operate recursively on directories +. +.SH "BUGS" +Report bugs at: https://github\.com/lz4/lz4/issues +. +.SH "AUTHOR" Yann Collet diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 64ce428..f10680d 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -28,8 +28,7 @@ The native file format is the `.lz4` format. ### Difference between lz4 and gzip -`lz4` supports a command line syntax similar -_but not identical_ to `gzip(1)`. +`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. Differences are : * `lz4` preserves original files @@ -44,7 +43,11 @@ Differences are : (use `-q` to silent them) Default behaviors can be modified by opt-in commands, described below. -`lz4 --quiet --multiple` more closely mimics `gzip` behavior. +`lz4 -m` more closely follows `gzip` command line syntax, +supporting compression of multiple files into `.lz4`, +the last remaining difference being that source files are preserved by default. +Since it's also possible to erase source file using opt-in command `--rm`, +`lz4 -m --rm` behaves the same as `gzip`. ### Concatenation of .lz4 files @@ -134,6 +137,7 @@ only the latest one will be applied. With `-m`, it is possible to specify any number of input filenames. Each of them will be compressed independently, and the resulting name of each compressed file will be `filename.lz4`. + This mode also reduces notification level. * `-B#`: Block size \[4-7\](default : 7)
-- cgit v0.12 From 6b90cfbeab05fb2a74b613352a038330b83da61a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 03:43:01 -0800 Subject: Updated man page --- programs/lz4.1 | 42 ++++++++++++++++++++++++++-------- programs/lz4.1.md | 68 +++++++++++++++++++++++++++++++------------------------ 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/programs/lz4.1 b/programs/lz4.1 index d73a7ed..48c988a 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -26,24 +26,38 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4\fR preserves original files . .IP "\(bu" 4 -\fBlz4\fR compresses a single file by default (use \fB\-m\fR for multiple files) +\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) . .IP "\(bu" 4 \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 . .IP "\(bu" 4 -When no destination name is provided, compressed file name receives a \fB\.lz4\fR suffix +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) . .IP "\(bu" 4 -When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB\-c\fR)\. Therefore \fBlz4 file > /dev/null\fR will not create \fBfile\.lz4\fR +If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. . .IP "\(bu" 4 -\fBlz4 file\fR shows real\-time statistics during compression (use \fB\-q\fR to silent them) +If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\. . .IP "" 0 . .P -Default behaviors can be modified by opt\-in commands, described below\. \fBlz4 \-m\fR more closely follows \fBgzip\fR command line syntax, supporting compression of multiple files into \fB\.lz4\fR, the last remaining difference being that source files are preserved by default\. Since it\'s also possible to erase source file using opt\-in command \fB\-\-rm\fR, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +Default behaviors can be modified by opt\-in commands, detailed below\. +. +.IP "\(bu" 4 +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\. +. +.IP "\(bu" 4 +It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.IP "" 0 . .SS "Concatenation of \.lz4 files" It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 @@ -102,7 +116,11 @@ Force write to standard output, even if it is the console\. . .TP \fB\-m\fR \fB\-\-multiple\fR -Multiple file names\. By default, the second filename is used as the destination filename for the compressed file\. With \fB\-m\fR, it is possible to specify any number of input filenames\. Each of them will be compressed independently, and the resulting name of each compressed file will be \fBfilename\.lz4\fR\. This mode also reduces notification level\. +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. +. +.TP +\fB\-r\fR +operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. . .TP \fB\-B#\fR @@ -113,7 +131,7 @@ Block size [4\-7](default : 7) . .TP \fB\-BD\fR -Block dependency (improves compression ratio on small blocks) +Block Dependency (improves compression ratio on small blocks) . .TP \fB\-\-[no\-]frame\-crc\fR @@ -132,10 +150,10 @@ Sparse mode support (default:enabled on file, disabled on stdout) . .TP \fB\-l\fR -Use Legacy format (typically used for Linux Kernel compression) +Use Legacy format (typically for Linux Kernel compression) . .br -Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR . .SS "Other options" . @@ -157,7 +175,11 @@ Display Version number and exit . .TP \fB\-k\fR \fB\-\-keep\fR -Don\'t delete source file\. This is default behavior anyway, so this option is just for compatibility with \fBgzip(1)\fR / \fBxz(1)\fR\. +Preserve source files (default behavior) +. +.TP +\fB\-\-rm\fR +Delete source files on successful compression or decompression . .SS "Benchmark mode" . diff --git a/programs/lz4.1.md b/programs/lz4.1.md index f10680d..71ad326 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -32,22 +32,29 @@ The native file format is the `.lz4` format. Differences are : * `lz4` preserves original files - * `lz4` compresses a single file by default (use `-m` for multiple files) + * `lz4` compresses a single file by default (see `-m` for multiple files) * `lz4 file1 file2` means : compress file1 _into_ file2 - * When no destination name is provided, compressed file name receives - a `.lz4` suffix - * When no destination name is provided, if `stdout` is _not_ the console, - it becomes the output (like a silent `-c`). - Therefore `lz4 file > /dev/null` will not create `file.lz4` - * `lz4 file` shows real-time statistics during compression - (use `-q` to silent them) - -Default behaviors can be modified by opt-in commands, described below. -`lz4 -m` more closely follows `gzip` command line syntax, -supporting compression of multiple files into `.lz4`, -the last remaining difference being that source files are preserved by default. -Since it's also possible to erase source file using opt-in command `--rm`, -`lz4 -m --rm` behaves the same as `gzip`. + * `lz4` shows real-time notification statistics + during compression or decompression of a single file + (use `-q` to silent them) + * If no destination name is provided, result is sent to `stdout` + _except if stdout is the console_. + * If no destination name is provided, __and__ if `stdout` is the console, + `file` is compressed into `file.lz4`. + * As a consequence of previous rules, note the following example : + `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, + hence it does _not_ create any `file.lz4`. + +Default behaviors can be modified by opt-in commands, detailed below. + + * `lz4 -m` makes it possible to provide multiple input filenames, + which will be compressed into files using suffix `.lz4`. + Progress notifications are also disabled by default. + This mode has a behavior which more closely mimics `gzip` command line, + with the main difference being that source files are preserved by default. + * It's possible to opt-in to erase source files + on successful compression or decompression, using `--rm` command. + * Consequently, `lz4 -m --rm` behaves the same as `gzip`. ### Concatenation of .lz4 files @@ -131,20 +138,22 @@ only the latest one will be applied. Force write to standard output, even if it is the console. * `-m` `--multiple`: - Multiple file names. - By default, the second filename is used as the destination filename - for the compressed file. - With `-m`, it is possible to specify any number of input filenames. - Each of them will be compressed independently, and the resulting name of - each compressed file will be `filename.lz4`. + Multiple input files. + Compressed file names will be appended a `.lz4` suffix. This mode also reduces notification level. + `lz4 -m` has a behavior equivalent to `gzip -k` + (it preserves source files by default). + +* `-r` : + operate recursively on directories. + This mode also sets `-m` (multiple input files). * `-B#`: Block size \[4-7\](default : 7)
`-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB * `-BD`: - Block dependency (improves compression ratio on small blocks) + Block Dependency (improves compression ratio on small blocks) * `--[no-]frame-crc`: Select frame checksum (default:enabled) @@ -159,8 +168,8 @@ only the latest one will be applied. Sparse mode support (default:enabled on file, disabled on stdout) * `-l`: - Use Legacy format (typically used for Linux Kernel compression)
- Note : `-l` is not compatible with `-m` (`--multiple`) + Use Legacy format (typically for Linux Kernel compression)
+ Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` ### Other options @@ -168,8 +177,8 @@ only the latest one will be applied. Verbose mode * `-q` `--quiet`: - Suppress warnings and real-time statistics; specify twice to suppress - errors too + Suppress warnings and real-time statistics; + specify twice to suppress errors too * `-h` `-H` `--help`: Display help/long help and exit @@ -178,9 +187,10 @@ only the latest one will be applied. Display Version number and exit * `-k` `--keep`: - Don't delete source file. - This is default behavior anyway, so this option is just for compatibility - with `gzip(1)` / `xz(1)`. + Preserve source files (default behavior) + +* `--rm` : + Delete source files on successful compression or decompression ### Benchmark mode -- cgit v0.12 From a9a8dbf7c10125f6024943df2e84108de49ce8e8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 04:01:02 -0800 Subject: fixed minor display error when using -r --- NEWS | 3 +++ lib/lz4.h | 2 +- programs/lz4.1.md | 3 --- programs/lz4cli.c | 45 ++++++++++++++++++++++++--------------------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index f6874ce..56a2875 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v1.7.5 +cli : fix minor notification when using -r recursive mode + v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/lib/lz4.h b/lib/lz4.h index 7420ad8..0aae19c 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -85,7 +85,7 @@ extern "C" { /*========== Version =========== */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 71ad326..83a08b6 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -204,9 +204,6 @@ only the latest one will be applied. * `-i#`: Minimum evaluation in seconds \[1-9\] (default : 3) -* `-r`: - Operate recursively on directories - BUGS ---- diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 71378aa..a452d1c 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -129,19 +129,20 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output *****************************/ static int usage(const char* exeName) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "Usage : \n"); + DISPLAY( " %s [arg] [input] [output] \n", exeName); DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); + DISPLAY( "input : a filename \n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); + DISPLAY( "Arguments : \n"); DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); + DISPLAY( " -z : force compression \n"); DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -h/-H : display help/long help and exit \n"); return 0; } @@ -151,33 +152,33 @@ static int usage_advanced(const char* exeName) usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -V : display Version number and exit \n"); + DISPLAY( " -v : verbose mode \n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); #ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); + DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); + DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [4-7] (default: 7)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); + DISPLAY( "Legacy arguments : \n"); + DISPLAY( " -c0 : fast compression \n"); + DISPLAY( " -c1 : high compression \n"); + DISPLAY( " -hc : high compression \n"); DISPLAY( " -y : overwrite output without prompting \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ EXTENDED_HELP; @@ -548,7 +549,7 @@ int main(int argc, const char** argv) } /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) { + while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ size_t const inSize = strlen(input_filename); @@ -583,6 +584,8 @@ int main(int argc, const char** argv) break; } + if (!output_filename) output_filename = "*\\dummy^!//"; + /* Check if output is defined as console; trigger an error in this case */ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); -- cgit v0.12 From 9a6f7dfc5b1e9ce882944c45de895cf9c21d1b7c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 04:05:37 -0800 Subject: updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 56a2875..b879469 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v1.7.5 cli : fix minor notification when using -r recursive mode +doc : markdown version of man page, by Takayuki Matsuoka v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From 5eb547a6080b8e63b5e522b448e31052fbc6b927 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 4 Dec 2016 20:05:36 +0100 Subject: fix #283 : implement LZ4_versionString(). --- lib/lz4.c | 1 + programs/lz4cli.c | 2 +- tests/fuzzer.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index fc19493..143c36e 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -391,6 +391,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } diff --git a/programs/lz4cli.c b/programs/lz4cli.c index a452d1c..46c2f1b 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -78,7 +78,7 @@ ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR +#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR #define LZ4_EXTENSION ".lz4" #define LZ4CAT "lz4cat" #define UNLZ4 "unlz4" diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 06cfd40..260208a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1102,7 +1102,7 @@ int main(int argc, const char** argv) } } - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString()); if (!seedset) { time_t const t = time(NULL); -- cgit v0.12 From 3f430daf7aa6ce0a96a863387de33be54d45806e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 15:20:59 +0100 Subject: added lz4opt.h --- lib/lz4opt.h | 573 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 573 insertions(+) create mode 100644 lib/lz4opt.h diff --git a/lib/lz4opt.h b/lib/lz4opt.h new file mode 100644 index 0000000..990cc67 --- /dev/null +++ b/lib/lz4opt.h @@ -0,0 +1,573 @@ +/* + lz4opt.h - Optimal Mode of LZ4 + Copyright (C) 2015-2016, Przemyslaw Skibinski + Note : this file is intended to be included within lz4hc.c + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) + + +#define LZ4_OPT_NUM (1<<12) + +typedef struct +{ + int off; + int len; + int back; +} LZ4HC_match_t; + +typedef struct +{ + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + + +FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) +{ + size_t price = 8*litlen; + if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + return price; +} + +FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t offset, size_t mlen) +{ + size_t price = 16 + 8; /* 16-bit offset + token */ + + price += 8*litlen; + if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + + // mlen-=MINMATCH; + if (mlen>=(int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + + return price; +} + + +FORCE_INLINE int LZ4HC_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ4HC_match_t* matches) +{ + U16* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + U32 matchIndex; + int nbAttempts = ctx->searchNum; + int mnum = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ4HC_hashPtr(ip)]; + matchIndex = *HashPos; + + + DELTANEXTU16(current) = (U16)(current - matchIndex); + *HashPos = current; + ctx->nextToUpdate++; + + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if ((ip[best_mlen] == match[best_mlen]) && (LZ4_read32(match) == LZ4_read32(ip))) + { + size_t mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + + while ((ip+back>iLowLimit) + && (match+back > lowPrefixPtr) + && (ip[back-1] == match[back-1])) + back--; + + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + matchIndex -= DELTANEXTU16(matchIndex); + } + + + return mnum; +} + + + +FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ4HC_match_t* matches) +{ + U16* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + int nbAttempts = ctx->searchNum; + int mnum = 0; + U16 *ptr0, *ptr1; + U32 matchIndex, delta0, delta1; + size_t mlt = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ4HC_hashPtr(ip)]; + matchIndex = *HashPos; + + *HashPos = current; + ctx->nextToUpdate++; + + // check rest of matches + ptr0 = &DELTANEXTU16(current*2+1); + ptr1 = &DELTANEXTU16(current*2); + delta0 = delta1 = current - matchIndex; + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + mlt = 0; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if (LZ4_read32(match) == LZ4_read32(ip)) + { + mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + + if (*(ip+mlt) < *(match+mlt)) + { + *ptr0 = delta0; + ptr0 = &DELTANEXTU16(matchIndex*2); + // printf("delta0=%d\n", delta0); + if (*ptr0 == (U16)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } + else + { + *ptr1 = delta1; + ptr1 = &DELTANEXTU16(matchIndex*2+1); + // printf("delta1=%d\n", delta1); + if (*ptr1 == (U16)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U16)-1; + *ptr1 = (U16)-1; + + return mnum; +} + + + + + +#define SET_PRICE(pos, mlen, offset, litlen, price) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + +static int LZ4HC_compress_optimal ( + LZ4HC_CCtx_internal* ctx, + const BYTE* source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive limit, + const int binaryTreeFinder, + const size_t sufficient_len, + const int faster_get_matches + ) +{ + LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; + LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2, skip_num = 0; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + + /* init */ + ctx->end += inputSize; + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + memset(opt, 0, sizeof(LZ4HC_optimal_t)); + last_pos = 0; + llen = ip - anchor; + + best_mlen = (last_pos) ? last_pos : (MINMATCH-1); + + if (faster_get_matches && last_pos) + match_num = 0; + else + { + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, ip); + match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); + } + } + + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (size_t)matches[match_num-1].len > sufficient_len) + { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto encode; + } + + // set prices using matches at position = 0 + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; + LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + while (mlen <= best_mlen) + { + litlen = 0; + price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + mlen++; + } + } + + if (last_pos < MINMATCH) { ip++; continue; } + + opt[0].mlen = opt[1].mlen = 1; + + // check further positions + for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) + { + litlen = opt[cur-1].litlen + 1; + + if (cur != litlen) + { + price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); + LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } + else + { + price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; + LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } + else + { + litlen = 1; + price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); + } + + mlen = 1; + best_mlen = 0; + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, mlen, best_mlen, litlen, price); + + if (cur == last_pos) break; + + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + + + if (faster_get_matches && skip_num > 0) + { + skip_num--; + continue; + } + + + best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); + + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, inr); + match_num = LZ4HC_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + + + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) + { + cur -= matches[match_num-1].back; + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto encode; + } + + // set prices using matches at position = cur + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + cur2 = cur - matches[i].back; + best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; + LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + + if (mlen < (size_t)matches[i].back + 1) + mlen = matches[i].back + 1; + + while (mlen <= best_mlen) + { + if (opt[cur2].mlen == 1) + { + litlen = opt[cur2].litlen; + + if (cur2 != litlen) + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + else + price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + } + else + { + litlen = 0; + price = opt[cur2].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + } + + LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) + { + SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + } + + mlen++; + } + } + } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + +encode: // cur, last_pos, best_mlen, best_off have to be set + for (i = 1; i <= last_pos; i++) + { + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + } + + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + + opt[0].mlen = 1; + + while (1) + { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = (int)best_mlen; + opt[cur].off = (int)best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (i = 0; i <= last_pos;) + { + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + i += opt[i].mlen; + } + + cur = 0; + + while (cur < last_pos) + { + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + + LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); + LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); + + if (res) return 0; + + LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); + } + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + // if (inputSize > LASTLITERALS && lastRun < LASTLITERALS) { printf("ERROR: lastRun=%d\n", lastRun); } + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun< Date: Tue, 6 Dec 2016 15:21:28 +0100 Subject: introduced LZ4HC_compress_hashChain --- lib/lz4hc.c | 30 +++++++++++++++++++++++++++--- lib/lz4hc.h | 8 +++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 298550c..57587f3 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -305,8 +305,9 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } +#include "lz4opt.h" -static int LZ4HC_compress_generic ( +static int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, @@ -336,8 +337,6 @@ static int LZ4HC_compress_generic ( const BYTE* ref0; /* init */ - if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL; - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; @@ -490,6 +489,31 @@ _Search3: } +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int const inputSize, + int const maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) +{ + if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; + if (compressionLevel > 16) { + switch (compressionLevel) { + case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64, 0); + case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256, 0); + case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64, 0); + case 20: + default: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 256, 0); + } + } + + return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limit); +} + + int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index d8ac3c0..40ca401 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,7 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 16 +#define LZ4HC_MAX_CLEVEL 20 /*-************************************ @@ -58,7 +58,7 @@ extern "C" { * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. - * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL. * @return : the number of bytes written into 'dst' * or 0 if compression fails. */ @@ -153,6 +153,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -169,12 +170,13 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; #endif -#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE 262200 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From 2113ead176e0032c7ba04aa93f3bcc3d04ba6142 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 19:11:53 +0100 Subject: 17-bit LZ4HC_MAXD --- lib/lz4hc.c | 16 ++++++++++------ lib/lz4hc.h | 4 ++-- lib/lz4opt.h | 51 ++++++++++++++++++--------------------------------- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 57587f3..abdef68 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,7 +84,7 @@ * Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -499,14 +499,18 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { +/* +16#silesia_tar : 211947520 -> 77841782 (2.723), 9.8 MB/s ,1874.4 MB/s + +*/ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 16) { switch (compressionLevel) { - case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64, 0); - case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256, 0); - case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64, 0); - case 20: - default: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 256, 0); + case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64); + case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256); + case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64); + default: + case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, LZ4_OPT_NUM); } } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 40ca401..4882d66 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -131,7 +131,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ #define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<nextToUpdate++; // check rest of matches - ptr0 = &DELTANEXTU16(current*2+1); - ptr1 = &DELTANEXTU16(current*2); + ptr0 = &DELTANEXTMAXD(current*2+1); + ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = current - matchIndex; while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) @@ -265,7 +265,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; - ptr0 = &DELTANEXTU16(matchIndex*2); + ptr0 = &DELTANEXTMAXD(matchIndex*2); // printf("delta0=%d\n", delta0); if (*ptr0 == (U16)-1) break; delta0 = *ptr0; @@ -275,7 +275,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( else { *ptr1 = delta1; - ptr1 = &DELTANEXTU16(matchIndex*2+1); + ptr1 = &DELTANEXTMAXD(matchIndex*2+1); // printf("delta1=%d\n", delta1); if (*ptr1 == (U16)-1) break; delta1 = *ptr1; @@ -313,14 +313,13 @@ static int LZ4HC_compress_optimal ( int maxOutputSize, limitedOutput_directive limit, const int binaryTreeFinder, - const size_t sufficient_len, - const int faster_get_matches + const size_t sufficient_len ) { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; const BYTE *inr; - size_t res, cur, cur2, skip_num = 0; + size_t res, cur, cur2; size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; const BYTE* ip = (const BYTE*) source; @@ -330,7 +329,6 @@ static int LZ4HC_compress_optimal ( const BYTE* const matchlimit = (iend - LASTLITERALS); BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - /* init */ ctx->end += inputSize; @@ -343,22 +341,17 @@ static int LZ4HC_compress_optimal ( last_pos = 0; llen = ip - anchor; - best_mlen = (last_pos) ? last_pos : (MINMATCH-1); + best_mlen = MINMATCH-1; - if (faster_get_matches && last_pos) - match_num = 0; - else - { - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, ip); - match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - } - } + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, ip); + match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); + } LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -393,7 +386,7 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; // check further positions - for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; @@ -430,14 +423,6 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - - if (faster_get_matches && skip_num > 0) - { - skip_num--; - continue; - } - - best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); if (!binaryTreeFinder) @@ -500,7 +485,7 @@ static int LZ4HC_compress_optimal ( mlen++; } } - } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + } // for (cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; -- cgit v0.12 From cfe6a24b0844dffa79f4a7749e076aadb4497f75 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 19:42:47 +0100 Subject: updated LZ4HC_get_price --- lib/lz4opt.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index fbd3048..5208438 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -63,7 +63,7 @@ FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) return price; } -FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t offset, size_t mlen) +FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ @@ -307,7 +307,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, - const BYTE* source, + const char* const source, char* dest, int inputSize, int maxOutputSize, @@ -374,7 +374,7 @@ static int LZ4HC_compress_optimal ( while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; @@ -465,14 +465,14 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); else - price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); } LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); -- cgit v0.12 From 757ef1d5fc710598c8a7325f9b227ccd96692187 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 11:44:17 +0100 Subject: removed LZ4HC_GetAllMatches --- lib/lz4hc.c | 8 ++-- lib/lz4opt.h | 142 +++++------------------------------------------------------ 2 files changed, 15 insertions(+), 135 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index abdef68..4381238 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -506,11 +506,11 @@ static int LZ4HC_compress_generic ( if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 16) { switch (compressionLevel) { - case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64); - case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256); - case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64); + case 17: ctx->searchNum = 16; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 16); + case 18: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 64); + case 19: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); default: - case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, LZ4_OPT_NUM); + case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); } } diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 5208438..0bffa3b 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -42,17 +42,17 @@ typedef struct { - int off; - int len; - int back; + int off; + int len; + int back; } LZ4HC_match_t; typedef struct { - int price; - int off; - int mlen; - int litlen; + int price; + int off; + int mlen; + int litlen; } LZ4HC_optimal_t; @@ -63,6 +63,7 @@ FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) return price; } + FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ @@ -77,107 +78,6 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) } -FORCE_INLINE int LZ4HC_GetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - size_t best_mlen, - LZ4HC_match_t* matches) -{ - U16* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); - const BYTE* const dictBase = ctx->dictBase; - const BYTE* match; - U32 matchIndex; - int nbAttempts = ctx->searchNum; - int mnum = 0; - U32* HashPos; - - if (ip + MINMATCH > iHighLimit) return 0; - - /* First Match */ - HashPos = &HashTable[LZ4HC_hashPtr(ip)]; - matchIndex = *HashPos; - - - DELTANEXTU16(current) = (U16)(current - matchIndex); - *HashPos = current; - ctx->nextToUpdate++; - - - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) - { - nbAttempts--; - if (matchIndex >= dictLimit) - { - match = base + matchIndex; - - if ((ip[best_mlen] == match[best_mlen]) && (LZ4_read32(match) == LZ4_read32(ip))) - { - size_t mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - int back = 0; - - while ((ip+back>iLowLimit) - && (match+back > lowPrefixPtr) - && (ip[back-1] == match[back-1])) - back--; - - mlt -= back; - - if (mlt > best_mlen) - { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - } - - if (best_mlen > LZ4_OPT_NUM) break; - } - } - else - { - match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { - size_t mlt; - int back=0; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if (mlt > best_mlen) - { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - } - - if (best_mlen > LZ4_OPT_NUM) break; - } - } - matchIndex -= DELTANEXTU16(matchIndex); - } - - - return mnum; -} - - - FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, @@ -312,7 +212,6 @@ static int LZ4HC_compress_optimal ( int inputSize, int maxOutputSize, limitedOutput_directive limit, - const int binaryTreeFinder, const size_t sufficient_len ) { @@ -342,16 +241,7 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; best_mlen = MINMATCH-1; - - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, ip); - match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - } + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -425,18 +315,8 @@ static int LZ4HC_compress_optimal ( best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, inr); - match_num = LZ4HC_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); - LZ4_LOG_PARSER("%d: LZ4HC_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } - + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { -- cgit v0.12 From e3fee94742f9027aa2d27fae6458ce285889552e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:16:33 +0100 Subject: LZ4HC_MAX_CLEVEL = 12 --- lib/lz4hc.c | 16 ++++++---------- lib/lz4hc.h | 4 ++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 4381238..608dea9 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -313,7 +313,7 @@ static int LZ4HC_compress_hashChain ( char* const dest, int const inputSize, int const maxOutputSize, - int compressionLevel, + unsigned maxNbAttempts, limitedOutput_directive limit ) { @@ -326,7 +326,6 @@ static int LZ4HC_compress_hashChain ( BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - unsigned maxNbAttempts; int ml, ml2, ml3, ml0; const BYTE* ref = NULL; const BYTE* start2 = NULL; @@ -337,7 +336,6 @@ static int LZ4HC_compress_hashChain ( const BYTE* ref0; /* init */ - maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; ip++; @@ -504,17 +502,15 @@ static int LZ4HC_compress_generic ( */ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; - if (compressionLevel > 16) { + if (compressionLevel > 9) { switch (compressionLevel) { - case 17: ctx->searchNum = 16; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 16); - case 18: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 64); - case 19: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); + case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); + case 11: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); default: - case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); + case 12: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); } } - - return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limit); + return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 4882d66..fc8067b 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,7 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 20 +#define LZ4HC_MAX_CLEVEL 12 /*-************************************ @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE+2) #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) -- cgit v0.12 From 77b051ed7bf383b97db43e4fc4b523df3b003e8a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:17:24 +0100 Subject: improve code formatting --- lib/lz4hc.c | 6 ++-- lib/lz4opt.h | 100 ++++++++++++++++++++--------------------------------------- 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 608dea9..9dc6be1 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -498,8 +498,10 @@ static int LZ4HC_compress_generic ( ) { /* -16#silesia_tar : 211947520 -> 77841782 (2.723), 9.8 MB/s ,1874.4 MB/s - + 9#silesia_tar : 211947520 -> 77891907 (2.721), 24.3 MB/s ,2142.1 MB/s +10#silesia_tar : 211947520 -> 77841782 (2.723), 11.4 MB/s ,2185.3 MB/s +11#silesia_tar : 211947520 -> 77408334 (2.738), 6.1 MB/s ,2288.9 MB/s +12#silesia_tar : 211947520 -> 77319973 (2.741), 3.3 MB/s ,2361.0 MB/s */ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 9) { diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 0bffa3b..2eaba4f 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -105,7 +105,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( /* First Match */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; - *HashPos = current; ctx->nextToUpdate++; @@ -114,20 +113,15 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = current - matchIndex; - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) - { + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; mlt = 0; if (matchIndex >= dictLimit) { match = base + matchIndex; - - if (LZ4_read32(match) == LZ4_read32(ip)) - { + if (LZ4_read32(match) == LZ4_read32(ip)) { mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - - if (mlt > best_mlen) - { + if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); matches[mnum].len = (int)mlt; @@ -141,16 +135,14 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( else { match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { + if (LZ4_read32(match) == LZ4_read32(ip)) { const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - if (mlt > best_mlen) - { + if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); matches[mnum].len = (int)mlt; @@ -162,21 +154,18 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } } - if (*(ip+mlt) < *(match+mlt)) - { + if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - // printf("delta0=%d\n", delta0); + /* printf("delta0=%d\n", delta0); */ if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; - } - else - { + } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - // printf("delta1=%d\n", delta1); + /* printf("delta1=%d\n", delta1); */ if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; @@ -234,8 +223,7 @@ static int LZ4HC_compress_optimal ( ip++; /* Main Loop */ - while (ip < mflimit) - { + while (ip < mflimit) { memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; @@ -256,13 +244,11 @@ static int LZ4HC_compress_optimal ( } // set prices using matches at position = 0 - for (i = 0; i < match_num; i++) - { + for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); - while (mlen <= best_mlen) - { + while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) @@ -276,40 +262,33 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; // check further positions - for (cur = 1; cur <= last_pos; cur++) - { - inr = ip + cur; + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; - if (opt[cur-1].mlen == 1) - { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur != litlen) - { + if (cur != litlen) { price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } - else - { + } else { price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } - } - else - { + } else { litlen = 1; price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); - } + } - mlen = 1; - best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + mlen = 1; + best_mlen = 0; + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); - if (cur == last_pos) break; + if (cur == last_pos) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); @@ -318,8 +297,7 @@ static int LZ4HC_compress_optimal ( match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) - { + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -328,8 +306,7 @@ static int LZ4HC_compress_optimal ( } // set prices using matches at position = cur - for (i = 0; i < match_num; i++) - { + for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; cur2 = cur - matches[i].back; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; @@ -338,27 +315,22 @@ static int LZ4HC_compress_optimal ( if (mlen < (size_t)matches[i].back + 1) mlen = matches[i].back + 1; - while (mlen <= best_mlen) - { - if (opt[cur2].mlen == 1) - { + while (mlen <= best_mlen) { + if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 != litlen) price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); else price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; - } - else - { + } else { litlen = 0; price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); } LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) - if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) - { + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } @@ -373,8 +345,7 @@ static int LZ4HC_compress_optimal ( cur = last_pos - best_mlen; encode: // cur, last_pos, best_mlen, best_off have to be set - for (i = 1; i <= last_pos; i++) - { + for (i = 1; i <= last_pos; i++) { LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } @@ -382,8 +353,7 @@ encode: // cur, last_pos, best_mlen, best_off have to be set opt[0].mlen = 1; - while (1) - { + while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; opt[cur].mlen = (int)best_mlen; @@ -394,16 +364,14 @@ encode: // cur, last_pos, best_mlen, best_off have to be set cur -= mlen; } - for (i = 0; i <= last_pos;) - { + for (i = 0; i <= last_pos;) { LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; - while (cur < last_pos) - { + while (cur < last_pos) { LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } -- cgit v0.12 From f2ebf37bfe64e24a6f8833dfdd206266cae25f6f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:49:38 +0100 Subject: slightly improved ratio --- lib/lz4opt.h | 110 ++++++++++++++++++++++------------------------------------- 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 2eaba4f..cc51ec5 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -44,7 +44,6 @@ typedef struct { int off; int len; - int back; } LZ4HC_match_t; typedef struct @@ -56,7 +55,7 @@ typedef struct } LZ4HC_optimal_t; -FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) +FORCE_INLINE size_t LZ4HC_GetLiteralsPrice(size_t litlen) { size_t price = 8*litlen; if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } @@ -69,10 +68,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) size_t price = 16 + 8; /* 16-bit offset + token */ price += 8*litlen; - if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } - // mlen-=MINMATCH; - if (mlen>=(int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + mlen -= MINMATCH; + if (mlen >= (int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } return price; } @@ -119,37 +118,28 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (matchIndex >= dictLimit) { match = base + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - } - + mlt = LZ4_count(ip, match, iHighLimit); + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; if (best_mlen > LZ4_OPT_NUM) break; } } else { match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - } - + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip, match, vLimit); + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; if (best_mlen > LZ4_OPT_NUM) break; } } @@ -227,30 +217,26 @@ static int LZ4HC_compress_optimal ( memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches); + if (!match_num) { ip++; continue; } + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); - best_mlen = MINMATCH-1; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } - - if (match_num && (size_t)matches[match_num-1].len > sufficient_len) - { + if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; last_pos = 1; goto encode; - } + } - // set prices using matches at position = 0 - for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + /* set prices using matches at position = 0 */ + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; @@ -261,44 +247,38 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; - // check further positions + /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur != litlen) { - price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); + price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; + price = LZ4HC_GetLiteralsPrice(llen + litlen) - 8*llen; LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); + price = opt[cur - 1].price + LZ4HC_GetLiteralsPrice(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_GetLiteralsPrice(litlen)); } mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); - - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { - cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; last_pos = cur + 1; @@ -307,39 +287,33 @@ static int LZ4HC_compress_optimal ( // set prices using matches at position = cur for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; - cur2 = cur - matches[i].back; + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; + cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); - if (mlen < (size_t)matches[i].back + 1) - mlen = matches[i].back + 1; - while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); + price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); - // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } - mlen++; } } } // for (cur = 1; cur <= last_pos; cur++) - best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; @@ -352,7 +326,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); opt[0].mlen = 1; - while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; @@ -370,7 +343,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set } cur = 0; - while (cur < last_pos) { LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; -- cgit v0.12 From 1833be1cf0e545f8b24bb4a786e274a9e419e280 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:59:05 +0100 Subject: fixed gcc warnings --- lib/lz4hc.c | 2 +- lib/lz4opt.h | 16 +++++++--------- programs/lz4io.c | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 9dc6be1..de7a55c 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,7 +84,7 @@ * Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } diff --git a/lib/lz4opt.h b/lib/lz4opt.h index cc51ec5..6922e35 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -94,8 +94,8 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( const BYTE* match; int nbAttempts = ctx->searchNum; int mnum = 0; - U16 *ptr0, *ptr1; - U32 matchIndex, delta0, delta1; + U16 *ptr0, *ptr1, delta0, delta1; + U32 matchIndex; size_t mlt = 0; U32* HashPos; @@ -129,8 +129,8 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } else { - match = dictBase + matchIndex; const BYTE* vLimit = ip + (dictLimit - matchIndex); + match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip, match, vLimit); if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) @@ -245,9 +245,8 @@ static int LZ4HC_compress_optimal ( if (last_pos < MINMATCH) { ip++; continue; } - opt[0].mlen = opt[1].mlen = 1; - /* check further positions */ + opt[0].mlen = opt[1].mlen = 1; for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; @@ -285,7 +284,7 @@ static int LZ4HC_compress_optimal ( goto encode; } - // set prices using matches at position = cur + /* set prices using matches at position = cur */ for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; @@ -312,13 +311,13 @@ static int LZ4HC_compress_optimal ( mlen++; } } - } // for (cur = 1; cur <= last_pos; cur++) + } /* for (cur = 1; cur <= last_pos; cur++) */ best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; -encode: // cur, last_pos, best_mlen, best_off have to be set +encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } @@ -363,7 +362,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - // if (inputSize > LASTLITERALS && lastRun < LASTLITERALS) { printf("ERROR: lastRun=%d\n", lastRun); } if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Wed, 7 Dec 2016 13:07:39 +0100 Subject: improved LZ4HC_BinTree_GetAllMatches --- lib/lz4opt.h | 66 ++++++++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 6922e35..a2a9d0b 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -110,40 +110,30 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( // check rest of matches ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); - delta0 = delta1 = current - matchIndex; + delta0 = delta1 = (U16)(current - matchIndex); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; - mlt = 0; - if (matchIndex >= dictLimit) - { + if (matchIndex >= dictLimit) { match = base + matchIndex; mlt = LZ4_count(ip, match, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - mnum++; - if (best_mlen > LZ4_OPT_NUM) break; - } - } - else - { + } else { const BYTE* vLimit = ip + (dictLimit - matchIndex); match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip, match, vLimit); if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - mnum++; - if (best_mlen > LZ4_OPT_NUM) break; - } } - + + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; + if (best_mlen > LZ4_OPT_NUM) break; + } + if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); @@ -170,18 +160,15 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } - - - -#define SET_PRICE(pos, mlen, offset, litlen, price) \ - { \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = (int)mlen; \ - opt[pos].off = (int)offset; \ - opt[pos].litlen = (int)litlen; \ - opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ - } +#define SET_PRICE(pos, mlen, offset, litlen, price) \ +{ \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ +} static int LZ4HC_compress_optimal ( @@ -194,11 +181,11 @@ static int LZ4HC_compress_optimal ( const size_t sufficient_len ) { - LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; - LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; - const BYTE *inr; - size_t res, cur, cur2; - size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; + LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; @@ -373,4 +360,3 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ /* End */ return (int) ((char*)op-dest); } - -- cgit v0.12 From 1289038240ab81aee4cc70ef071da907ccfdfcde Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 14:31:46 +0100 Subject: 15-bit LZ4HC_HASH_LOG --- lib/lz4hc.h | 2 +- programs/lz4io.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index fc8067b..0523314 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE+2) +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) diff --git a/programs/lz4io.c b/programs/lz4io.c index f8a8f2b..46daaf7 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -591,9 +591,10 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t ress; if (dstFileName == NULL) return ifntSize; /* not enough memory */ + ress = LZ4IO_createCResources(); /* loop on each file */ for (i=0; i Date: Wed, 7 Dec 2016 15:49:45 +0100 Subject: improved ratio --- lib/lz4hc.h | 2 +- lib/lz4opt.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 0523314..5eafcd6 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -131,7 +131,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ #define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) +#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) #define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index a2a9d0b..1ff3375 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -224,7 +224,7 @@ static int LZ4HC_compress_optimal ( while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; - if (mlen > last_pos || price < (size_t)opt[mlen].price) + if (mlen > last_pos || price <= (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } @@ -255,7 +255,7 @@ static int LZ4HC_compress_optimal ( mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price < (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; -- cgit v0.12 From 76228f064a95f7135e5990387b23002dd296b2da Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 16:19:10 +0100 Subject: LZ4HC_DICTIONARY_LOGSIZE 17 --- lib/lz4hc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 5eafcd6..4ff1d17 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -130,11 +130,11 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE)) +#define LZ4HC_DICTIONARY_LOGSIZE 17 +#define LZ4HC_MAXD (1< Date: Wed, 7 Dec 2016 16:24:35 +0100 Subject: fixed LZ4_STREAMHCSIZE --- lib/lz4hc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 4ff1d17..35e88b6 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -176,7 +176,7 @@ typedef struct #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 52) /* 393268 */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From 01ffed25adbb4d18e48ceac59b1e5174ae29fc7f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 14:29:22 -0800 Subject: fix : invocation of path/to/unlz4 (#284), reported by @beiDei8z --- programs/Makefile | 21 ++++++++++++--------- programs/lz4cli.c | 9 ++++++++- tests/Makefile | 15 +++++++++++++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index ffbcbf1..84662c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,7 +91,7 @@ lz4c32: lz4 clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 @echo Cleaning completed @@ -112,13 +112,16 @@ preview-man: clean-man man #FreeBSD targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +unlz4: lz4 + ln -s lz4 unlz4 + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 + @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @@ -127,10 +130,10 @@ install: lz4$(EXT) lz4c$(EXT) @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/lz4cat + @$(RM) $(DESTDIR)$(BINDIR)/unlz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4c @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 46c2f1b..eda2018 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -260,6 +260,13 @@ static void waitEnter(void) (void)getchar(); } +static const char* lastNameFromPath(const char* path) +{ + const char* name = strrchr(path, '/'); + if (name==NULL) name = strrchr(path, '\\'); /* windows */ + if (name==NULL) name = path-1; + return name+1; +} /*! readU32FromChar() : @return : unsigned integer value reach from input in `char` format @@ -294,7 +301,7 @@ int main(int argc, const char** argv) const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = argv[0]; + const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; diff --git a/tests/Makefile b/tests/Makefile index e58d449..5923a20 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -206,12 +206,15 @@ test-lz4-multiple: lz4 datagen $(LZ4) -f -m tmp1 notHere tmp2; echo $$? @$(RM) tmp* -test-lz4-basic: lz4 datagen +unlz4: + @$(MAKE) -C $(PRGDIR) unlz4 + +test-lz4-basic: lz4 datagen unlz4 @echo "\n ---- test lz4 basic compression/decompression ----" ./datagen -g0 | $(LZ4) -v | $(LZ4) -t ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t ./datagen -g20KB > tmpSrc - $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes + $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes diff -q tmpSrc tmpRes $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes diff -q tmpSrc tmpRes @@ -220,6 +223,14 @@ test-lz4-basic: lz4 datagen ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + @echo "hello world" > tmp + $(LZ4) --rm tmp + ls -ls tmp && false || true # must fail (--rm) + ls -ls tmp.lz4 + $(PRGDIR)/unlz4 --rm tmp.lz4 + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (--rm) + ls -ls tmp.lz4.lz4 && false || true # must fail (unlz4) @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 750a50cf67e8574500e4f9b4bb5e7f6cdd6f9c26 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 14:49:04 -0800 Subject: fix #284, reported by @beiDei8z : added test cases for path/to/lz4cat --- programs/.gitignore | 2 ++ programs/Makefile | 5 ++++- programs/lz4cli.c | 1 + tests/Makefile | 14 +++++++++++--- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/programs/.gitignore b/programs/.gitignore index df3423e..daa7f14 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,5 +1,7 @@ # local binary (Makefile) lz4 +unlz4 +lz4cat lz4c lz4c32 datagen diff --git a/programs/Makefile b/programs/Makefile index 84662c2..74a2440 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,7 +91,7 @@ lz4c32: lz4 clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat @echo Cleaning completed @@ -115,6 +115,9 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) unlz4: lz4 ln -s lz4 unlz4 +lz4cat: lz4 + ln -s lz4 lz4cat + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index eda2018..76e7e14 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -319,6 +319,7 @@ int main(int argc, const char** argv) if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); + LZ4IO_setRemoveSrcFile(0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; diff --git a/tests/Makefile b/tests/Makefile index 5923a20..adfcfea 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -207,9 +207,12 @@ test-lz4-multiple: lz4 datagen @$(RM) tmp* unlz4: - @$(MAKE) -C $(PRGDIR) unlz4 + @$(MAKE) -C $(PRGDIR) $@ -test-lz4-basic: lz4 datagen unlz4 +lz4cat: + @$(MAKE) -C $(PRGDIR) $@ + +test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" ./datagen -g0 | $(LZ4) -v | $(LZ4) -t ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t @@ -224,13 +227,18 @@ test-lz4-basic: lz4 datagen unlz4 ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t @echo "hello world" > tmp - $(LZ4) --rm tmp + $(LZ4) --rm -f tmp ls -ls tmp && false || true # must fail (--rm) ls -ls tmp.lz4 + $(PRGDIR)/lz4cat tmp.lz4 # must display hello world + ls -ls tmp.lz4 $(PRGDIR)/unlz4 --rm tmp.lz4 ls -ls tmp ls -ls tmp.lz4 && false || true # must fail (--rm) ls -ls tmp.lz4.lz4 && false || true # must fail (unlz4) + $(PRGDIR)/lz4cat tmp # pass-through mode + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (lz4cat) @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 7e2d4c4c08934a1fa85fa8324c3e134ce69c6648 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 15:52:07 -0800 Subject: fixed test-lz4c32 --- tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index adfcfea..eb36abf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -38,7 +38,7 @@ VOID := /dev/null TESTDIR := versionsTest PYTHON ?= python3 -CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make +CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 @@ -207,10 +207,10 @@ test-lz4-multiple: lz4 datagen @$(RM) tmp* unlz4: - @$(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" lz4cat: - @$(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" -- cgit v0.12 From 0280cf40dd01caa784d291480aaead9987ae9f9a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 02:47:42 +0100 Subject: fixed path-1, as suggested by @t-mat --- programs/lz4cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 76e7e14..b63d69d 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -264,7 +264,7 @@ static const char* lastNameFromPath(const char* path) { const char* name = strrchr(path, '/'); if (name==NULL) name = strrchr(path, '\\'); /* windows */ - if (name==NULL) name = path-1; + if (name==NULL) return path; return name+1; } -- cgit v0.12 From fb6c98c856d300b39baa40f8408ab639e6e9b56a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 8 Dec 2016 15:50:45 +0100 Subject: slightly improved lz4opt.h --- lib/lz4hc.h | 2 +- lib/lz4opt.h | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 35e88b6..59d912e 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); return price; } @@ -68,10 +69,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) size_t price = 16 + 8; /* 16-bit offset + token */ price += 8*litlen; - if (litlen >= (int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); mlen -= MINMATCH; - if (mlen >= (int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + if (mlen >= (int)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); return price; } @@ -107,7 +108,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( *HashPos = current; ctx->nextToUpdate++; - // check rest of matches ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); @@ -137,7 +137,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - /* printf("delta0=%d\n", delta0); */ if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; @@ -145,7 +144,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - /* printf("delta1=%d\n", delta1); */ if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; @@ -155,7 +153,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; - return mnum; } @@ -223,9 +220,8 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; - if (mlen > last_pos || price <= (size_t)opt[mlen].price) - SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } } @@ -243,7 +239,7 @@ static int LZ4HC_compress_optimal ( price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4HC_GetLiteralsPrice(llen + litlen) - 8*llen; + price = LZ4HC_GetLiteralsPrice(llen + litlen) - LZ4HC_GetLiteralsPrice(llen); LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { @@ -255,7 +251,7 @@ static int LZ4HC_compress_optimal ( mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price < (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; @@ -285,14 +281,14 @@ static int LZ4HC_compress_optimal ( if (cur2 != litlen) price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; + price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); } else { litlen = 0; price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); } LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); - if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } mlen++; -- cgit v0.12 From de93e9e5d84fef6c5741ff0d72306fdb2baade1e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 14:17:40 -0800 Subject: fix #285 : lz4cat doesn't work with stdin (reported by @beiDei8z) --- programs/lz4.1.md | 8 +++++--- programs/lz4cli.c | 13 ++++++++----- tests/Makefile | 5 +++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 83a08b6..48f3152 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -34,6 +34,7 @@ Differences are : * `lz4` preserves original files * `lz4` compresses a single file by default (see `-m` for multiple files) * `lz4 file1 file2` means : compress file1 _into_ file2 + * `lz4 file.lz4` will default to decompression (use `-z` to force compression) * `lz4` shows real-time notification statistics during compression or decompression of a single file (use `-q` to silent them) @@ -43,15 +44,16 @@ Differences are : `file` is compressed into `file.lz4`. * As a consequence of previous rules, note the following example : `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, - hence it does _not_ create any `file.lz4`. + hence it does _not_ create `file.lz4`. Default behaviors can be modified by opt-in commands, detailed below. * `lz4 -m` makes it possible to provide multiple input filenames, which will be compressed into files using suffix `.lz4`. - Progress notifications are also disabled by default. + Progress notifications are also disabled by default (use `-v` to enable them). This mode has a behavior which more closely mimics `gzip` command line, - with the main difference being that source files are preserved by default. + with the main remaining difference being that source files are preserved by default. + * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. * It's possible to opt-in to erase source files on successful compression or decompression, using `--rm` command. * Consequently, `lz4 -m --rm` behaves the same as `gzip`. diff --git a/programs/lz4cli.c b/programs/lz4cli.c index b63d69d..5bd06d9 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -313,6 +313,7 @@ int main(int argc, const char** argv) DISPLAY("Allocation error : not enough memory \n"); return 1; } + inFileNames[0] = stdinmark; LZ4IO_setOverwrite(0); /* lz4cat predefined behavior */ @@ -468,7 +469,7 @@ int main(int argc, const char** argv) break; #ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ + /* recursive */ case 'r': recursive=1; /* without break */ #endif /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ @@ -482,7 +483,7 @@ int main(int argc, const char** argv) iters = readU32FromChar(&argument); argument--; BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ } break; @@ -555,6 +556,9 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + /* if input==stdin and no output defined, stdout becomes default output */ + if (!strcmp(input_filename, stdinmark) && !output_filename) + output_filename = stdoutmark; /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { @@ -592,20 +596,19 @@ int main(int argc, const char** argv) break; } - if (!output_filename) output_filename = "*\\dummy^!//"; - /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); exit(1); } - /* Downgrade notification level in stdout and multiple file mode */ if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); diff --git a/tests/Makefile b/tests/Makefile index eb36abf..01b45bc 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -239,6 +239,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat $(PRGDIR)/lz4cat tmp # pass-through mode ls -ls tmp ls -ls tmp.lz4 && false || true # must fail (lz4cat) + $(LZ4) tmp # creates tmp.lz4 + $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) + diff -q tmp tmp3 + $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) + diff -q tmp tmp2 @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 626552671a69a13cd467e27ed149dac4b35fd484 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 14:37:12 -0800 Subject: updated NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index b879469..292cd27 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v1.7.5 cli : fix minor notification when using -r recursive mode -doc : markdown version of man page, by Takayuki Matsuoka +lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +doc : markdown version of man page, by Takayuki Matsuoka (#279) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From a22e71d4a9f4bfd0fe12acc00a7fa286faad2da6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 12:37:17 +0100 Subject: full binary tree update --- lib/lz4hc.c | 6 ------ lib/lz4opt.h | 56 ++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index de7a55c..d3578b9 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -497,12 +497,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { -/* - 9#silesia_tar : 211947520 -> 77891907 (2.721), 24.3 MB/s ,2142.1 MB/s -10#silesia_tar : 211947520 -> 77841782 (2.723), 11.4 MB/s ,2185.3 MB/s -11#silesia_tar : 211947520 -> 77408334 (2.738), 6.1 MB/s ,2288.9 MB/s -12#silesia_tar : 211947520 -> 77319973 (2.741), 3.3 MB/s ,2361.0 MB/s -*/ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 9) { switch (compressionLevel) { diff --git a/lib/lz4opt.h b/lib/lz4opt.h index dc778c9..7e8150f 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -37,7 +37,6 @@ #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) - #define LZ4_OPT_NUM (1<<12) @@ -78,7 +77,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) } -FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( +/*-************************************* +* Binary Tree search +***************************************/ +FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, @@ -97,7 +99,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( int mnum = 0; U16 *ptr0, *ptr1, delta0, delta1; U32 matchIndex; - size_t mlt = 0; + size_t matchLength = 0; U32* HashPos; if (ip + MINMATCH > iHighLimit) return 0; @@ -106,7 +108,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; *HashPos = current; - ctx->nextToUpdate++; ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); @@ -116,25 +117,28 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( nbAttempts--; if (matchIndex >= dictLimit) { match = base + matchIndex; - mlt = LZ4_count(ip, match, iHighLimit); + matchLength = LZ4_count(ip, match, iHighLimit); } else { const BYTE* vLimit = ip + (dictLimit - matchIndex); match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip, match, vLimit); - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + matchLength = LZ4_count(ip, match, vLimit); + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); } - if (mlt > best_mlen) { - best_mlen = mlt; + if (matches && matchLength > best_mlen) { + best_mlen = matchLength; matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; + matches[mnum].len = (int)matchLength; mnum++; if (best_mlen > LZ4_OPT_NUM) break; } - if (*(ip+mlt) < *(match+mlt)) { + if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + + if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); if (*ptr0 == (U16)-1) break; @@ -157,6 +161,34 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } +FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen) +{ + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + + while(idx < target) { + LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, best_mlen, NULL); + idx++; + } +} + + +/** Tree updater, providing best match */ +FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, const BYTE* const iHighLimit, + size_t best_mlen, LZ4HC_match_t* matches) +{ + int mnum; + if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ + LZ4HC_updateBinTree(ctx, ip, iHighLimit, best_mlen); + mnum = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches); + ctx->nextToUpdate = (U32)(ip - ctx->base)+1; + return mnum; +} + + #define SET_PRICE(pos, mlen, offset, litlen, price) \ { \ while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ -- cgit v0.12 From d694bf91111e6227878ceda1131255f8eb5299c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 15:20:32 +0100 Subject: faster opt compression --- lib/lz4opt.h | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 7e8150f..dbc7556 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -85,7 +85,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen, - LZ4HC_match_t* matches) + LZ4HC_match_t* matches, + int* matchNum) { U16* const chainTable = ctx->chainTable; U32* const HashTable = ctx->hashTable; @@ -102,7 +103,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; - if (ip + MINMATCH > iHighLimit) return 0; + if (ip + MINMATCH > iHighLimit) return 1; /* First Match */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; @@ -127,11 +128,13 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); } - if (matches && matchLength > best_mlen) { + if (matchLength > best_mlen) { best_mlen = matchLength; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)matchLength; - mnum++; + if (matches) { + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)matchLength; + mnum++; + } if (best_mlen > LZ4_OPT_NUM) break; } @@ -157,20 +160,21 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; - return mnum; + if (matchNum) *matchNum = mnum; + // if (best_mlen > 8) return best_mlen-8; + if (!matchNum) return 8; + return 1; } -FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen) +FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdate; - while(idx < target) { - LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, best_mlen, NULL); - idx++; - } + while(idx < target) + idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -180,11 +184,11 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen, LZ4HC_match_t* matches) { - int mnum; + int mnum = 0; if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - LZ4HC_updateBinTree(ctx, ip, iHighLimit, best_mlen); - mnum = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches); - ctx->nextToUpdate = (U32)(ip - ctx->base)+1; + LZ4HC_updateBinTree(ctx, ip, iHighLimit); + best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); + ctx->nextToUpdate = (U32)(ip - ctx->base) + best_mlen; return mnum; } -- cgit v0.12 From b3ecc3e3ec768e89eea29248bd8ab0a23ddab2dd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 16:09:38 +0100 Subject: last match starts at least 12 bytes before end of block --- lib/lz4opt.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index dbc7556..c466677 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -36,6 +36,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#include #define LZ4_OPT_NUM (1<<12) @@ -161,8 +162,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; - // if (best_mlen > 8) return best_mlen-8; - if (!matchNum) return 8; + // if (best_mlen > 8) return best_mlen-8; + if (!matchNum) return 1; return 1; } @@ -290,7 +291,7 @@ static int LZ4HC_compress_optimal ( if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); - if (cur == last_pos) break; + if (cur == last_pos || inr >= mflimit) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); @@ -368,7 +369,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ offset = opt[cur].off; cur += mlen; - LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen); res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); @@ -379,12 +380,11 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ } /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); + { int lastRun = (int)(iend - anchor); if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Fri, 9 Dec 2016 17:16:35 +0100 Subject: fullUpdate as a parameter --- lib/lz4hc.c | 11 +++++++---- lib/lz4hc.h | 2 ++ lib/lz4opt.h | 23 ++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index d3578b9..e1eadf7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; + hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; @@ -501,9 +501,9 @@ static int LZ4HC_compress_generic ( if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); + case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); + case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -575,13 +575,15 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, @@ -641,6 +643,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit; } return dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 59d912e..a3736f8 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -153,6 +153,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t nextToUpdateBT; /* index from which to continue binary tree update */ uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -170,6 +171,7 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int nextToUpdateBT; /* index from which to continue binary tree update */ unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; diff --git a/lib/lz4opt.h b/lib/lz4opt.h index c466677..0844f8e 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -106,7 +106,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (ip + MINMATCH > iHighLimit) return 1; - /* First Match */ + /* HC4 match finder */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; *HashPos = current; @@ -162,7 +162,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; - // if (best_mlen > 8) return best_mlen-8; + /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; return 1; } @@ -172,24 +172,24 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; + U32 idx = ctx->nextToUpdateBT; while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); -} +} /** Tree updater, providing best match */ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, - size_t best_mlen, LZ4HC_match_t* matches) + size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) { int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - LZ4HC_updateBinTree(ctx, ip, iHighLimit); + if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ + if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdate = (U32)(ip - ctx->base) + best_mlen; + ctx->nextToUpdateBT = (U32)(ip - ctx->base) + best_mlen; return mnum; } @@ -212,7 +212,8 @@ static int LZ4HC_compress_optimal ( int inputSize, int maxOutputSize, limitedOutput_directive limit, - const size_t sufficient_len + const size_t sufficient_len, + const int fullUpdate ) { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; @@ -238,7 +239,7 @@ static int LZ4HC_compress_optimal ( memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); @@ -294,7 +295,7 @@ static int LZ4HC_compress_optimal ( if (cur == last_pos || inr >= mflimit) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { -- cgit v0.12 From 4f050b68d817776e3c1fac6c449b5f2b2adfbcb9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 18:17:46 +0100 Subject: tree update dependent from compression level --- lib/lz4hc.c | 6 ++++-- lib/lz4hc.h | 1 + lib/lz4opt.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e1eadf7..44a1340 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -574,8 +574,10 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + if (ctxPtr->compressionLevel >= LZ4HC_MIN_CLEVEL_OPT) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index a3736f8..7ce5958 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,6 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 +#define LZ4HC_MIN_CLEVEL_OPT 11 #define LZ4HC_MAX_CLEVEL 12 diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 0844f8e..f487edf 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -36,7 +36,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) -#include + #define LZ4_OPT_NUM (1<<12) @@ -189,7 +189,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdateBT = (U32)(ip - ctx->base) + best_mlen; + ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen); return mnum; } -- cgit v0.12 From 913b98fd6fe814ee4c719a90a0642497c9d3d0dc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Dec 2016 22:13:36 +0100 Subject: fix `make` concurrency build (#277) --- Makefile | 4 +++- NEWS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 48e6752..c77f697 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,9 @@ endif .PHONY: default all lib lz4 clean test versionsTest examples -default: lib lz4-release +default: + @$(MAKE) -C $(LZ4DIR) + @$(MAKE) -C $(PRGDIR) all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/NEWS b/NEWS index 292cd27..ff62ca8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v1.7.5 cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) doc : markdown version of man page, by Takayuki Matsuoka (#279) +build : Makefile : fix concurrency lib+exe build (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From 45b14ab8e0584adb97cb8d9366687f623bdd9123 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:03:13 +0100 Subject: added platfom.h --- programs/platform.h | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 programs/platform.h diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 0000000..562c831 --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ +#endif +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ + || defined __x86_64__ || defined _M_X64 /* x86 64-bit */ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ || + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */) +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ************************************** +* Unix Large Files support (>4GB) +****************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + + + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) + /* UNIX-style OS. ------------------------------------------- */ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif + +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ -- cgit v0.12 From bb69cdf2583a56c6c703f5b91aae0632912f5a8a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:03:30 +0100 Subject: updated util.h --- programs/util.h | 128 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/programs/util.h b/programs/util.h index 1ad61bc..7bcc4c4 100644 --- a/programs/util.h +++ b/programs/util.h @@ -20,6 +20,7 @@ - LZ4 source repository : https://github.com/lz4/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -27,70 +28,64 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif - /*-**************************************** * Dependencies ******************************************/ -#include /* features.h with _POSIX_C_SOURCE, malloc */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include /* malloc */ #include /* fprintf */ -#include /* strerr, strlen, memcpy */ -#include /* ptrdiff_t */ #include /* stat, utime */ #include /* stat */ #if defined(_MSC_VER) - #include /* utime */ - #include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else - #include /* chown, stat */ - #include /* utime */ +# include /* chown, stat */ +# include /* utime */ #endif #include /* time */ #include -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) +/*-************************************ +* OS-specific Includes +**************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif #else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) +# define fseek fseeko /* fseeko() added in FreeBSD 3.2 */ +# endif +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) #endif @@ -102,7 +97,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -112,7 +107,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -124,6 +119,12 @@ extern "C" { #endif +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + /*-************************************************************** * Basic Types *****************************************************************/ @@ -148,6 +149,20 @@ extern "C" { /*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline +#else +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** * Time functions ******************************************/ #if !defined(_WIN32) @@ -362,8 +377,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(__APPLE__) && defined(__MACH__)) || \ - ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ @@ -427,7 +441,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } @@ -443,16 +457,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -474,8 +487,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i Date: Tue, 20 Dec 2016 18:05:43 +0100 Subject: executables use platform.h --- programs/bench.c | 5 +++-- programs/lz4cli.c | 24 ++---------------------- programs/lz4io.c | 33 ++------------------------------- tests/fullbench.c | 16 +--------------- 4 files changed, 8 insertions(+), 70 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 434da8b..f077b42 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -27,10 +27,11 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ -#include /* fprintf, fopen, ftello64 */ +#include /* fprintf, fopen, ftello */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include "datagen.h" /* RDG_genBuffer */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 5bd06d9..e552de7 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -39,17 +39,11 @@ /* #define ENABLE_LZ4C_LEGACY_OPTIONS */ -/************************************** -* Compiler Options -***************************************/ -/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */ -#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within on unix */ - - /**************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST */ #include /* fprintf, getchar */ #include /* exit, calloc, free */ #include /* strcmp, strlen */ @@ -59,20 +53,6 @@ #include "lz4.h" /* LZ4_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /***************************** * Constants ******************************/ diff --git a/programs/lz4io.c b/programs/lz4io.c index 46daaf7..a9e5515 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -33,10 +33,6 @@ /************************************** * Compiler Options **************************************/ -#define _LARGE_FILES /* Large file support on 32-bits AIX */ -#define _FILE_OFFSET_BITS 64 /* off_t width */ -#define _LARGEFILE_SOURCE - #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif @@ -44,7 +40,8 @@ /***************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_getFileStat */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_getFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ #include /* strcmp, strlen */ @@ -57,32 +54,6 @@ #include "lz4frame.h" -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - /***************************** * Constants diff --git a/tests/fullbench.c b/tests/fullbench.c index 7d54492..df6d4fc 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -23,20 +23,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif // S_ISREG & gettimeofday() are not supported by MSVC #if defined(_MSC_VER) || defined(_WIN32) @@ -48,7 +34,7 @@ * Includes **************************************/ #include /* malloc, free */ -#include /* fprintf, fopen, ftello64 */ +#include /* fprintf, fopen, ftello */ #include /* stat64 */ #include /* stat64 */ #include /* strcmp */ -- cgit v0.12 From 6adf05d1ed83053b8bb1f762494d2c10fdd8ac1d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:56:09 +0100 Subject: improved platform.h --- programs/datagen.c | 13 +------ programs/lz4cli.c | 4 +-- programs/lz4io.c | 4 +-- programs/platform.h | 100 +++++++++++++++++++++++++++++++++++----------------- programs/util.h | 41 --------------------- 5 files changed, 72 insertions(+), 90 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index f97c2d0..f44e310 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,6 +26,7 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ #include /* malloc */ #include /* FILE, fwrite */ #include /* memcpy */ @@ -51,18 +52,6 @@ /************************************** -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -#else -# define SET_BINARY_MODE(file) -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index e552de7..0b0fbd4 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -42,8 +42,8 @@ /**************************** * Includes *****************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ -#include "util.h" /* UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* fprintf, getchar */ #include /* exit, calloc, free */ #include /* strcmp, strlen */ diff --git a/programs/lz4io.c b/programs/lz4io.c index a9e5515..506d661 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -40,8 +40,8 @@ /***************************** * Includes *****************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_getFileStat */ +#include "platform.h" /* Compiler options, Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ #include /* strcmp, strlen */ diff --git a/programs/platform.h b/programs/platform.h index 562c831..df70037 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -24,9 +24,53 @@ extern "C" { # define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif +# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 #endif @@ -48,51 +92,41 @@ extern "C" { #endif -/* ************************************** -* Unix Large Files support (>4GB) -****************************************/ -#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ # define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ # endif # if defined(_AIX) || defined(__hpux) # define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ # endif +# define fseek fseeko #endif - - - -/* ************************************************************ -* Detect POSIX version -* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX -* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION -***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) - /* UNIX-style OS. ------------------------------------------- */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ -# define PLATFORM_POSIX_VERSION 200112L +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # else -# if defined(__linux__) || defined(__linux) -# define _POSIX_C_SOURCE 200112L /* use feature test macro */ -# endif -# include /* declares _POSIX_VERSION */ -# if defined(_POSIX_VERSION) /* POSIX compliant */ -# define PLATFORM_POSIX_VERSION _POSIX_VERSION -# else -# define PLATFORM_POSIX_VERSION 0 -# endif +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) # endif -#endif - -#if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION -1 +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) #endif diff --git a/programs/util.h b/programs/util.h index 7bcc4c4..5ec1a4a 100644 --- a/programs/util.h +++ b/programs/util.h @@ -48,47 +48,6 @@ extern "C" { #include -/*-************************************ -* OS-specific Includes -**************************************/ -#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko /* fseeko() added in FreeBSD 3.2 */ -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - - /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ -- cgit v0.12 From 2efedaf7dc8f20d2e5761ee600712bdb0371bc5c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 19:31:05 +0100 Subject: fixed gcc warnings --- programs/platform.h | 2 +- programs/util.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/platform.h b/programs/platform.h index df70037..244b2d8 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -116,9 +116,9 @@ extern "C" { # include /* _O_BINARY */ # include /* _setmode, _fileno, _get_osfhandle */ # if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } # include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } # define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # else # define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) diff --git a/programs/util.h b/programs/util.h index 5ec1a4a..b0529b0 100644 --- a/programs/util.h +++ b/programs/util.h @@ -34,6 +34,7 @@ extern "C" { ******************************************/ #include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ #include /* malloc */ +#include /* size_t, ptrdiff_t */ #include /* fprintf */ #include /* stat, utime */ #include /* stat */ @@ -339,6 +340,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ +# include /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { -- cgit v0.12 From 5f0cc8e2d0dc473185ebac61b03fb6a4262797fb Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 20:52:18 +0100 Subject: fixed gcc warnings (2) --- programs/lz4io.c | 4 +++- programs/platform.h | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 506d661..a69b80f 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -53,7 +53,9 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" - +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko +#endif /***************************** * Constants diff --git a/programs/platform.h b/programs/platform.h index 244b2d8..e983756 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -41,7 +41,7 @@ extern "C" { ***************************************************************/ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ # define PLATFORM_POSIX_VERSION 200112L # else @@ -105,7 +105,6 @@ extern "C" { # if defined(_AIX) || defined(__hpux) # define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ # endif -# define fseek fseeko #endif -- cgit v0.12 From e1332241a586cf95476fc19b2f6ba14b7b685863 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 21:11:39 +0100 Subject: improved formatting --- programs/platform.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index e983756..7e004ca 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -39,10 +39,10 @@ extern "C" { * PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX * PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION ***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) @@ -78,14 +78,14 @@ extern "C" { * Detect 64-bit OS * http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros ****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ - || defined __x86_64__ || defined _M_X64 /* x86 64-bit */ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ || - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */) +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ # if !defined(__64BIT__) # define __64BIT__ 1 # endif -- cgit v0.12 From 090cb887a3fb2b3974b1d0fd228ae08ddc6a2329 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 22:21:03 +0100 Subject: improved formatting (2) --- programs/lz4io.c | 2 ++ programs/platform.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index a69b80f..15c69f0 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -53,9 +53,11 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" +#if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif +#endif /***************************** * Constants diff --git a/programs/platform.h b/programs/platform.h index 7e004ca..ae9587e 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -84,7 +84,7 @@ extern "C" { || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ # if !defined(__64BIT__) # define __64BIT__ 1 -- cgit v0.12 From 452cc970d6b35601a2501dab47050592c97bc4ed Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 23:38:32 +0100 Subject: reordered definitons in platform.h --- programs/lz4cli.c | 9 +++++++ programs/platform.h | 69 +++++++++++++++++++++++++++-------------------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0b0fbd4..366aed5 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -498,6 +498,15 @@ int main(int argc, const char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { diff --git a/programs/platform.h b/programs/platform.h index ae9587e..b8911f2 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -14,6 +14,7 @@ extern "C" { #endif + /* ************************************** * Compiler Options ****************************************/ @@ -33,6 +34,40 @@ extern "C" { #endif +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + /* ************************************************************ * Detect POSIX version * PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows @@ -74,40 +109,6 @@ extern "C" { #endif -/* ************************************** -* Detect 64-bit OS -* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros -****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ - || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ -# if !defined(__64BIT__) -# define __64BIT__ 1 -# endif -#endif - - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - - /****************************** * OS-specific Includes ******************************/ -- cgit v0.12 From 9546ba62d01a9618aab91eafe77929120653d275 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:44:59 +0100 Subject: executables use basic types from util.h --- programs/datagen.c | 20 +------------------- programs/platform.h | 26 ++++++++++++++++++-------- programs/util.h | 11 +++++------ tests/Makefile | 2 +- tests/datagencli.c | 20 +------------------- tests/frametest.c | 22 ++-------------------- tests/fullbench.c | 46 ++-------------------------------------------- tests/fuzzer.c | 24 ++---------------------- 8 files changed, 32 insertions(+), 139 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index f44e310..a61afc0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -27,31 +27,13 @@ * Includes **************************************/ #include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ #include /* malloc */ #include /* FILE, fwrite */ #include /* memcpy */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/platform.h b/programs/platform.h index b8911f2..c4b97c8 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,11 +1,21 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ #ifndef PLATFORM_H_MODULE #define PLATFORM_H_MODULE diff --git a/programs/util.h b/programs/util.h index b0529b0..409f6d7 100644 --- a/programs/util.h +++ b/programs/util.h @@ -15,10 +15,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #ifndef UTIL_H_MODULE @@ -88,7 +84,9 @@ extern "C" { /*-************************************************************** * Basic Types *****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#ifndef BASIC_TYPES_DEFINED +#define BASIC_TYPES_DEFINED +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -105,7 +103,8 @@ extern "C" { typedef signed int S32; typedef unsigned long long U64; typedef signed long long S64; -#endif +#endif +#endif /*-**************************************** diff --git a/tests/Makefile b/tests/Makefile index 01b45bc..3e39608 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -43,7 +43,7 @@ CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-e -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_ +CPPFLAGS:= -I$(LIBDIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) diff --git a/tests/datagencli.c b/tests/datagencli.c index 0b78d37..c985197 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -27,31 +27,13 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32 */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ #include "lz4.h" /* LZ4_VERSION_STRING */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..ba971d6 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -27,7 +27,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -35,6 +34,8 @@ /*-************************************ * Includes **************************************/ +#include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include /* malloc, free */ #include /* fprintf */ #include /* strcmp */ @@ -45,25 +46,6 @@ #include "xxhash.h" /* XXH64 */ -/*-************************************ -* Basic Types -**************************************/ -#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - /* unoptimized version; solves endianess & alignment issues */ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) { diff --git a/tests/fullbench.c b/tests/fullbench.c index df6d4fc..8d55d2d 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -33,6 +33,7 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32, UTIL_getFileSize */ #include /* malloc, free */ #include /* fprintf, fopen, ftello */ #include /* stat64 */ @@ -48,34 +49,6 @@ /************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define PROGRAM_DESCRIPTION "LZ4 speed analyzer" @@ -180,21 +153,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - /********************************************************* * Benchmark function *********************************************************/ @@ -384,7 +342,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory size adjustments */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 260208a..18ba7c8 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,8 +27,6 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif @@ -37,6 +35,8 @@ /*-************************************ * Dependencies **************************************/ +#include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include #include /* fgets, sscanf */ #include /* strcmp */ @@ -47,26 +47,6 @@ /*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ -#endif - - -/*-************************************ * Constants **************************************/ #define NB_ATTEMPTS (1<<16) -- cgit v0.12 From 64cbc4e1dc17492d89915a9d774dc15ee438ccfe Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 10:22:40 +0100 Subject: improved MinGW support --- programs/lz4cli.c | 3 +++ programs/lz4io.c | 6 ------ programs/platform.h | 3 +++ tests/frametest.c | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 366aed5..e03aa98 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -507,6 +507,9 @@ int main(int argc, const char** argv) #ifdef PLATFORM_POSIX_VERSION DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif +#ifdef _FILE_OFFSET_BITS + DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { diff --git a/programs/lz4io.c b/programs/lz4io.c index 15c69f0..df3ed41 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -30,12 +30,6 @@ - The license of this source file is GPLv2. */ -/************************************** -* Compiler Options -**************************************/ -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif /***************************** * Includes diff --git a/programs/platform.h b/programs/platform.h index c4b97c8..02ebd4c 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -42,6 +42,9 @@ extern "C" { # define fseek _fseeki64 # endif #endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ +#endif /* ************************************** diff --git a/tests/frametest.c b/tests/frametest.c index ba971d6..8b7caba 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -92,7 +92,7 @@ static clock_t g_clockTime = 0; *****************************************/ static U32 no_prompt = 0; static U32 displayLevel = 2; -static U32 pause = 0; +static U32 use_pause = 0; /*-******************************************************* @@ -705,7 +705,7 @@ _end: free(compressedBuffer); free(decodedBuffer); - if (pause) { + if (use_pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } @@ -777,7 +777,7 @@ int main(int argc, const char** argv) break; case 'p': /* pause at the end */ argument++; - pause = 1; + use_pause = 1; break; case 'i': -- cgit v0.12 From 3c6ce9c66526f0f6e3c200ddb64db2c66be5e031 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 10:39:27 +0100 Subject: fixed gcc 4.4 support --- tests/fuzzer.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 18ba7c8..10e9139 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -47,6 +47,14 @@ /*-************************************ +* Basic Types +**************************************/ +#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ +#endif + + +/*-************************************ * Constants **************************************/ #define NB_ATTEMPTS (1<<16) @@ -980,7 +988,7 @@ int main(int argc, const char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; + int use_pause = 0; const char* programName = argv[0]; U32 duration = 0; @@ -992,7 +1000,7 @@ int main(int argc, const char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; } argument++; while (*argument!=0) { @@ -1008,7 +1016,7 @@ int main(int argc, const char** argv) case 'p': /* pause at the end */ argument++; - pause=1; + use_pause=1; break; case 'i': @@ -1098,7 +1106,7 @@ int main(int argc, const char** argv) if (nbTests<=0) nbTests=1; { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { + if (use_pause) { DISPLAY("press enter ... \n"); (void)getchar(); } -- cgit v0.12 From 2fd7eb554a17ce4815c60b29a226130e0ea448d6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:53:16 +0100 Subject: fixed Visual Studio compilation --- programs/lz4io.c | 8 ++++++++ programs/platform.h | 3 --- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index df3ed41..ca3cea5 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -47,12 +47,20 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +#endif + #if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif #endif + /***************************** * Constants *****************************/ diff --git a/programs/platform.h b/programs/platform.h index 02ebd4c..fe4bfde 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -38,9 +38,6 @@ extern "C" { # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif -# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ -- cgit v0.12 From 287f7d3d6d32e48a711ec8979e30a113469e235b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:58:50 +0100 Subject: added IncludePath --- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 363c5ae..76d12c9 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index 8d0a623..e2d95c9 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 6672aaa..85d6c9b 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true -- cgit v0.12 From 253ac12cb76c5bacab69fdc723982ae9a92fcf53 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 12:06:36 +0100 Subject: VS projects: CharacterSet changed to MultiByte --- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench-dll/fullbench-dll.vcxproj | 16 ++++++++-------- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 76d12c9..1998e89 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 16a4f0d..1ee263d 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index e2d95c9..f3734dd 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 85d6c9b..46a7c0f 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte -- cgit v0.12 From 4f61505cc40cb8b2c6c371600a427fd685c656a2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 12:09:58 +0100 Subject: datagen.vcxproj: CharacterSet is MultiByte --- visual/VS2010/datagen/datagen.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index aaf81ad..825bef6 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte -- cgit v0.12 From 385cb4f539134b3cb855a29eef99a4a876f728d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 13:18:02 +0100 Subject: minor update Makefile --- Makefile | 1 + lib/Makefile | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index c77f697..647ab4e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ endif default: @$(MAKE) -C $(LZ4DIR) @$(MAKE) -C $(PRGDIR) + @cp $(PRGDIR)/lz4$(EXT) . all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..8426330 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -71,6 +71,8 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +LIBLZ4 = liblz4.$(SHARED_EXT_VER) + default: lib-release lib-release: liblz4.a liblz4 @@ -90,18 +92,20 @@ ifeq ($(BUILD_STATIC),yes) @$(AR) rcs $@ *.o endif -liblz4: *.c +$(LIBLZ4): *.c @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib else - @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@ @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) + @ln -sf $@ liblz4.$(SHARED_EXT) endif +liblz4: $(LIBLZ4) + clean: @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib @echo Cleaning library completed @@ -123,21 +127,21 @@ liblz4.pc: liblz4.pc.in Makefile install: lib liblz4.pc @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @install -m 755 liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) + @install -m 755 liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR) endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR) @echo lz4 static and shared libraries installed uninstall: + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h -- cgit v0.12 From 7cf0bb97b2a988cb17435780d19e145147dd9f70 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 14:18:01 +0100 Subject: LZ4F_compressBound(0) provides upper bound for LZ4F_flush() and LZ4F_compressEnd() [#290, suggested by @vtermanis] --- lib/lz4frame.c | 15 +++++++++++---- lib/lz4frame.h | 23 ++++++++++++----------- tests/frametest.c | 7 +++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..626ac98 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ static const size_t BHSize = 4; @@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +/* LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; size_t const blockSize = LZ4F_getBlockSize(bid); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ @@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..a4a4ce6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /* Compression */ +#define LZ4F_HEADER_SIZE_MAX 15 LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. + * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) @@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ @@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..fd07377 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -204,6 +204,13 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); -- cgit v0.12 From bc0839c5df185129e899640e77a05f1c8fa595f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:00:43 +0100 Subject: minor fix for travis-install test --- Makefile | 4 ++-- NEWS | 5 +++-- contrib/cmake_unofficial/.gitignore | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 647ab4e..1432e6b 100644 --- a/Makefile +++ b/Makefile @@ -99,10 +99,10 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - $(MAKE) install PREFIX=~/install_test_dir + $(MAKE) -j1 install PREFIX=~/install_test_dir test: - $(MAKE) -C $(TESTDIR) test + $(MAKE) -C $(TESTDIR) $@ clangtest: clean clang -v diff --git a/NEWS b/NEWS index ff62ca8..e2f7666 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.7.5 -cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +cli : fix minor notification when using -r recursive mode +API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix concurrency lib+exe build (#277) +build : Makefile : fix make lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/contrib/cmake_unofficial/.gitignore b/contrib/cmake_unofficial/.gitignore index 0f81929..d39505d 100644 --- a/contrib/cmake_unofficial/.gitignore +++ b/contrib/cmake_unofficial/.gitignore @@ -5,3 +5,5 @@ CMakeFiles *.cmake Makefile liblz4.pc +lz4c +install_manifest.txt -- cgit v0.12 From 70526a11e033a640fb9e7ae80fe75b465d484bf0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:33:53 +0100 Subject: fixed lib/clean --- lib/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index 8426330..701dc07 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -107,7 +107,8 @@ endif liblz4: $(LIBLZ4) clean: - @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) @echo Cleaning library completed -- cgit v0.12 From 973bc79740f571c2b6700cab16a319df9aed2a05 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 16:10:09 +0100 Subject: util.h and platform.h based on zstd --- programs/lz4io.c | 6 ++++- programs/platform.h | 13 +++------- programs/util.h | 72 +++++++++++++++++++++++++++-------------------------- tests/frametest.c | 2 +- tests/fullbench.c | 1 + tests/fuzzer.c | 3 ++- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index ca3cea5..36498b2 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -31,10 +31,14 @@ */ +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + /***************************** * Includes *****************************/ -#include "platform.h" /* Compiler options, Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ diff --git a/programs/platform.h b/programs/platform.h index fe4bfde..f1040c0 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -25,23 +25,17 @@ extern "C" { #endif + /* ************************************** * Compiler Options ****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif #if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif #endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ -#endif /* ************************************** @@ -110,6 +104,7 @@ extern "C" { * Detect if isatty() and fileno() are available ************************************************/ #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ diff --git a/programs/util.h b/programs/util.h index 409f6d7..c670c20 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,26 +25,51 @@ extern "C" { #endif + /*-**************************************** * Dependencies ******************************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ -#include /* malloc */ -#include /* size_t, ptrdiff_t */ -#include /* fprintf */ -#include /* stat, utime */ -#include /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include /* malloc */ +#include /* size_t, ptrdiff_t */ +#include /* fprintf */ +#include /* stat, utime */ +#include /* stat */ #if defined(_MSC_VER) -# include /* utime */ -# include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else # include /* chown, stat */ # include /* utime */ #endif -#include /* time */ +#include /* time */ #include + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ @@ -81,35 +106,12 @@ extern "C" { #define LIST_SIZE_INCREASE (8*1024) -/*-************************************************************** -* Basic Types -*****************************************************************/ -#ifndef BASIC_TYPES_DEFINED -#define BASIC_TYPES_DEFINED -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif -#endif - - /*-**************************************** * Compiler specifics ******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif #if defined(__GNUC__) # define UTIL_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) diff --git a/tests/frametest.c b/tests/frametest.c index 8b7caba..869be76 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -27,6 +27,7 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -34,7 +35,6 @@ /*-************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ #include "util.h" /* U32 */ #include /* malloc, free */ #include /* fprintf */ diff --git a/tests/fullbench.c b/tests/fullbench.c index 8d55d2d..f489392 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -33,6 +33,7 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */ #include "util.h" /* U32, UTIL_getFileSize */ #include /* malloc, free */ #include /* fprintf, fopen, ftello */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 10e9139..ff02e0c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,6 +27,7 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif @@ -35,7 +36,7 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options */ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ #include "util.h" /* U32 */ #include #include /* fgets, sscanf */ -- cgit v0.12 From f084b653835940c749d4fc92a610e861a6644aa2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 17:13:38 +0100 Subject: test Large File support for Mac OS-X in 32-bits mode --- .travis.yml | 4 +++- programs/bench.c | 8 ++++++++ programs/lz4io.c | 13 +++++++++++-- tests/Makefile | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3f476b..dc61505 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + install: + - export CC=clang + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" && CFLAGS=-m32 make -C tests clean test-lz4-contentSize' COMPILER=clang # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux diff --git a/programs/bench.c b/programs/bench.c index f077b42..01e610c 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -24,6 +24,14 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ diff --git a/programs/lz4io.c b/programs/lz4io.c index 36498b2..3d69a6e 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -31,10 +31,17 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif + /***************************** * Includes *****************************/ @@ -644,8 +651,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer storedSkips += (unsigned)(nb0T * sizeT); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + errno = 0; + { int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; diff --git a/tests/Makefile b/tests/Makefile index 3e39608..b63493d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -173,7 +173,7 @@ test-lz4-contentSize: lz4 datagen $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 diff -s tmp tmp2 # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp + @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp @ls -ls tmp @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 @ls -ls tmp2 -- cgit v0.12 From 58124506358648b05477698e9136d7f651f11e01 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 18:46:14 +0100 Subject: use fseeko for 32-bit MacOS --- programs/lz4io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 3d69a6e..877c034 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -58,18 +58,16 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" + /* ************************************** * Compiler Options ****************************************/ #if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 #endif - -#if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif -#endif /***************************** -- cgit v0.12 From e6af952f8495de3a9e3cabb9a21e0ba4d7d28ff0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 19:49:06 +0100 Subject: improved comments --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 877c034..e8370ea 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -45,7 +45,7 @@ /***************************** * Includes *****************************/ -#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ -- cgit v0.12 From fea95c156761ee10d32a4f12105ebd90c67d4f47 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 22 Dec 2016 10:58:58 +0100 Subject: use FindFirstFileA and FindNextFileA on Windows --- programs/lz4io.c | 2 +- programs/util.h | 6 +++--- visual/VS2010/datagen/datagen.vcxproj | 8 ++++---- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench-dll/fullbench-dll.vcxproj | 8 ++++---- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- visual/VS2010/lz4/lz4.vcxproj | 8 ++++---- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index e8370ea..640c76d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -49,7 +49,7 @@ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ -#include /* strcmp, strlen */ +#include /* strerror, strcmp, strlen */ #include /* clock */ #include /* stat64 */ #include /* stat64 */ diff --git a/programs/util.h b/programs/util.h index c670c20..f3ff1b2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -283,7 +283,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -295,7 +295,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -332,7 +332,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index 825bef6..aaf81ad 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 1998e89..76d12c9 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 1ee263d..c10552a 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index f3734dd..e2d95c9 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 46a7c0f..85d6c9b 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj index 295a94b..693e121 100644 --- a/visual/VS2010/lz4/lz4.vcxproj +++ b/visual/VS2010/lz4/lz4.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode -- cgit v0.12 From 52cac9a97342641315c76cfb861206d6acd631a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:41:05 +0100 Subject: updated a few macros names --- lib/lz4frame.c | 14 +++++++------- lib/lz4hc.c | 4 ++-- lib/lz4hc.h | 8 ++++---- programs/bench.c | 6 +++--- programs/lz4cli.c | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 626ac98..a0a625b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -329,7 +329,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; } @@ -356,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -424,10 +424,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -452,7 +452,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -533,7 +533,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < LZ4HC_MIN_CLEVEL) { + if (level < LZ4HC_CLEVEL_MIN) { if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } @@ -543,7 +543,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 44a1340..ae245b5 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -497,7 +497,7 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); @@ -574,7 +574,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->compressionLevel >= LZ4HC_MIN_CLEVEL_OPT) + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 7ce5958..ad731d9 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -44,10 +44,10 @@ extern "C" { /* --- Useful constants --- */ -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MIN_CLEVEL_OPT 11 -#define LZ4HC_MAX_CLEVEL 12 +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 11 +#define LZ4HC_CLEVEL_MAX 12 /*-************************************ diff --git a/programs/bench.c b/programs/bench.c index 434da8b..406adcf 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -170,7 +170,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -494,8 +494,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 5bd06d9..80d84a1 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -208,7 +208,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); -- cgit v0.12 From 7eb16d97d238e2b98f1e6453ed6ad64058cb99c1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:50:21 +0100 Subject: updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index e2f7666..bb77e04 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.7.5 +lz4hc : new high compression mode : levels 10-12 compress more and slower, by @inikep. lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) -- cgit v0.12 From 9b4b081c151ddde70587f0838062a9fd660bcc49 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 22 Dec 2016 17:18:39 +0100 Subject: "make test" is now compatible with Solaris --- programs/Makefile | 6 +++--- tests/Makefile | 38 ++++++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 74a2440..5f14ad7 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -108,9 +108,9 @@ preview-man: clean-man man #------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#make install is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and +#Solaris targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) unlz4: lz4 ln -s lz4 unlz4 diff --git a/tests/Makefile b/tests/Makefile index b63493d..d23ed46 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -120,15 +120,21 @@ versionsTest: #------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and +#Solaris targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) MD5:=md5 -r endif +DIFF:=diff +ifneq (,$(filter $(shell uname),SunOS)) +DIFF:=gdiff +endif + + test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test32: CFLAGS+=-m32 @@ -138,18 +144,18 @@ test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" ./datagen -g5M -P100 > tmpSrc $(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4 - diff -s tmpSrc tmpB4 + $(DIFF) -s tmpSrc tmpB4 $(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5 - diff -s tmpSrc tmpB5 + $(DIFF) -s tmpSrc tmpB5 $(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6 - diff -s tmpSrc tmpB6 + $(DIFF) -s tmpSrc tmpB6 $(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7 - diff -s tmpSrc tmpB7 + $(DIFF) -s tmpSrc tmpB7 $(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse - diff -s tmpSrc tmpNoSparse + $(DIFF) -s tmpSrc tmpNoSparse ls -ls tmp* ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd + ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmpOdd ls -ls tmpOdd @$(RM) tmp* @echo "\n Compatibility with Console :" @@ -163,7 +169,7 @@ test-lz4-sparse: lz4 datagen $(LZ4) -d -v tmpC tmpR $(LZ4) -d -v tmpC >> tmpR ls -ls tmp* - diff tmp2M tmpR + $(DIFF) tmp2M tmpR @$(RM) tmp* test-lz4-contentSize: lz4 datagen @@ -171,13 +177,13 @@ test-lz4-contentSize: lz4 datagen ./datagen -g15M > tmp $(LZ4) -v tmp | $(LZ4) -t $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 # test large size [2-4] GB @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp @ls -ls tmp @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 @ls -ls tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 @$(RM) tmp* test-lz4-frame-concatenation: lz4 datagen @@ -218,9 +224,9 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t ./datagen -g20KB > tmpSrc $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(DIFF) -q tmpSrc tmpRes $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(DIFF) -q tmpSrc tmpRes ./datagen | $(LZ4) | $(LZ4) -t ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt @@ -241,9 +247,9 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat ls -ls tmp.lz4 && false || true # must fail (lz4cat) $(LZ4) tmp # creates tmp.lz4 $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) - diff -q tmp tmp3 + $(DIFF) -q tmp tmp3 $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) - diff -q tmp tmp2 + $(DIFF) -q tmp tmp2 @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 19c0f21b000ababf11cb38e0d6154742f5e1cd83 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 18:02:09 +0100 Subject: updated Makefile : lz4 no longer recompiled when already up-to-date --- NEWS | 2 +- lib/lz4opt.h | 59 ++++++++++++++++++++++++++++--------------------------- programs/Makefile | 29 ++++++++++++++++----------- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/NEWS b/NEWS index bb77e04..d149128 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @be cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix make lib+exe concurrency (#277) +build : Makefile : fix make -jX lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index f487edf..94b38e0 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -2,7 +2,7 @@ lz4opt.h - Optimal Mode of LZ4 Copyright (C) 2015-2016, Przemyslaw Skibinski Note : this file is intended to be included within lz4hc.c - + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -33,9 +33,9 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_OPT_NUM (1<<12) @@ -56,23 +56,24 @@ typedef struct } LZ4HC_optimal_t; -FORCE_INLINE size_t LZ4HC_GetLiteralsPrice(size_t litlen) +/* price in bits */ +FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) { size_t price = 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); return price; } -FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) +/* requires mlen >= MINMATCH */ +FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ - price += 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + price += LZ4HC_literalsPrice(litlen); mlen -= MINMATCH; - if (mlen >= (int)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); + if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); return price; } @@ -103,7 +104,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( U32 matchIndex; size_t matchLength = 0; U32* HashPos; - + if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -164,7 +165,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (matchNum) *matchNum = mnum; /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; - return 1; + return 1; } @@ -258,7 +259,7 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } @@ -274,18 +275,18 @@ static int LZ4HC_compress_optimal ( if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; if (cur != litlen) { - price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); + price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4HC_GetLiteralsPrice(llen + litlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LZ4HC_GetLiteralsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_GetLiteralsPrice(litlen)); + price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen)); } - + mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); @@ -293,7 +294,7 @@ static int LZ4HC_compress_optimal ( SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); @@ -317,12 +318,12 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); @@ -340,31 +341,31 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); opt[0].mlen = 1; while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; + opt[cur].mlen = (int)best_mlen; opt[cur].off = (int)best_off; best_mlen = mlen; best_off = offset; if (mlen > cur) break; cur -= mlen; } - + for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -374,7 +375,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); - if (res) return 0; + if (res) return 0; LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); } diff --git a/programs/Makefile b/programs/Makefile index 74a2440..6f4c8c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -43,7 +43,9 @@ PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib -VOID := /dev/null + +SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) +OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 @@ -60,33 +62,36 @@ MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +VOID := nul +EXT :=.exe else -EXT = +VOID := /dev/null +EXT := endif default: lz4-release -lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o lz4$(EXT) - all: lz4 lz4c all32: CFLAGS+=-m32 all32: all lz4: CFLAGS += $(DEBUGFLAGS) -lz4: lz4-release +lz4: $(OBJFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lz4-release: DEBUGFLAGS= +lz4-release: lz4 -lz4c: CFLAGS += $(DEBUGFLAGS) -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o +lz4c : CFLAGS += $(DEBUGFLAGS) +lz4c : $(SRCFILES) $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) +lz4c32: $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) -- cgit v0.12 From ea51ad0ceefd0a47df706a9301a8baf9b2629cc0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 23 Dec 2016 00:02:01 +0100 Subject: Solaris: working "make install" --- lib/Makefile | 53 +++++++++++++++++++++++++++++++++++------------------ programs/Makefile | 37 +++++++++++++++++++++++++------------ tests/Makefile | 2 +- 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..f9fad29 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -42,11 +42,6 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) BUILD_STATIC:= yes -DESTDIR ?= -PREFIX ?= /usr/local -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include - CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -107,10 +102,30 @@ clean: @echo Cleaning library completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +ifneq (,$(filter $(shell uname),SunOS)) +PREFIX ?= /usr +INSTALL ?= ginstall +else +PREFIX ?= /usr/local +INSTALL ?= install +endif +DESTDIR ?= +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) +PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +else +PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig +endif + +INSTALL_LIB ?= $(INSTALL) -m 755 +INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile @echo creating pkgconfig @@ -121,17 +136,19 @@ liblz4.pc: liblz4.pc.in Makefile $< >$@ install: lib liblz4.pc - @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ + @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ + @$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/ + @echo Installing libraries ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @echo Installing includes + @$(INSTALL_DATA) -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 static and shared libraries installed uninstall: diff --git a/programs/Makefile b/programs/Makefile index 5f14ad7..f91a326 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,10 +38,6 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib VOID := /dev/null @@ -107,10 +103,10 @@ preview-man: clean-man man man ./lz4.1 -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and -#Solaris targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) unlz4: lz4 ln -s lz4 unlz4 @@ -118,15 +114,32 @@ unlz4: lz4 lz4cat: lz4 ln -s lz4 lz4cat + +ifneq (,$(filter $(shell uname),SunOS)) +PREFIX ?= /usr +INSTALL ?= ginstall +else +PREFIX ?= /usr/local +INSTALL ?= install +endif +DESTDIR ?= +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 + + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c + @$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages - @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @$(INSTALL_MAN) -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 diff --git a/tests/Makefile b/tests/Makefile index d23ed46..1b92718 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,7 +122,7 @@ versionsTest: #------------------------------------------------------------------------ #make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and #Solaris targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) -- cgit v0.12 From b22e2bed5e44f2cdacb11cf01d4a15a00bcb76ca Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 23 Dec 2016 10:05:41 +0100 Subject: BSD: improved "make install" --- lib/Makefile | 17 +++++++++++------ programs/Makefile | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index f9fad29..13d0482 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -108,15 +108,20 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) ifneq (,$(filter $(shell uname),SunOS)) -PREFIX ?= /usr INSTALL ?= ginstall else -PREFIX ?= /usr/local INSTALL ?= install endif -DESTDIR ?= -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +PREFIX ?= /usr +else +PREFIX ?= /usr/local +endif + +DESTDIR ?= +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR ?= $(PREFIX)/include ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig @@ -124,7 +129,7 @@ else PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif -INSTALL_LIB ?= $(INSTALL) -m 755 +INSTALL_LIB ?= $(INSTALL) -m 755 INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile diff --git a/programs/Makefile b/programs/Makefile index f91a326..14b042a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -116,19 +116,24 @@ lz4cat: lz4 ifneq (,$(filter $(shell uname),SunOS)) -PREFIX ?= /usr INSTALL ?= ginstall else -PREFIX ?= /usr/local INSTALL ?= install endif + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +PREFIX ?= /usr +else +PREFIX ?= /usr/local +endif + DESTDIR ?= -BINDIR = $(PREFIX)/bin -MANDIR = $(PREFIX)/share/man/man1 +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/share/man/man1 INSTALL_PROGRAM ?= $(INSTALL) -m 755 -INSTALL_SCRIPT ?= $(INSTALL) -m 755 -INSTALL_MAN ?= $(INSTALL) -m 644 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 install: lz4$(EXT) lz4c$(EXT) -- cgit v0.12 From e6536faf337e1c4e202f32bec860e2c87511d7f3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 11:17:35 +0100 Subject: lib\Makefile: fixed INSTALL_DATA --- lib/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 50d4a41..3e21c1b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -156,9 +156,9 @@ endif @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @echo Installing includes - @$(INSTALL_DATA) -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @$(INSTALL_DATA) -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @$(INSTALL_DATA) -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 static and shared libraries installed uninstall: -- cgit v0.12 From f4575f4f148dd62894d5d2f0af2e8b1a5fc08ae2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 13:14:04 +0100 Subject: added test-lz4-opt-parser --- tests/Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 1b92718..bf84034 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -277,7 +277,15 @@ test-lz4-testmode: lz4 datagen $(LZ4) -fm file1-dne file2-dne && false || true $(LZ4) -fm file1-dne file2-dne && false || true -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \ +test-lz4-opt-parser: lz4 datagen + @echo "\n ---- test opt-parser ----" + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen | $(LZ4) -12 | $(LZ4) -t + ./datagen -g1M | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g256MB | $(LZ4) -11vqB4D | $(LZ4) -qt + +test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ test-lz4-hugefile -- cgit v0.12 From 20381a2fce8216b9d844c8da2ba3ecbd31118560 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 15:31:35 +0100 Subject: fixed -BD compression --- lib/lz4opt.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 94b38e0..46797ef 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -133,7 +133,10 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (matchLength > best_mlen) { best_mlen = matchLength; if (matches) { - matches[mnum].off = (int)(ip - match); + if (matchIndex >= dictLimit) + matches[mnum].off = (int)(ip - match); + else + matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ matches[mnum].len = (int)matchLength; mnum++; } -- cgit v0.12 From a3d61cf3dff74e286bb731ce6533c9992c49d4d9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 15:38:07 +0100 Subject: improved test-lz4-opt-parser --- tests/Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index bf84034..97fa782 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -279,11 +279,16 @@ test-lz4-testmode: lz4 datagen test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" - ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t - ./datagen | $(LZ4) -12 | $(LZ4) -t - ./datagen -g1M | $(LZ4) -11B5 | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -11vqB4D | $(LZ4) -qt + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t + ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t + ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t + ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t + ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt + ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t + ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ -- cgit v0.12 From d57ff6456d2fbf1c644eb07ff0684646bea6808a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 11:27:17 +0100 Subject: changed default PREFIX and MANDIR --- lib/Makefile | 7 +------ programs/Makefile | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 3e21c1b..2d9c8f3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,12 +118,7 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else -PREFIX ?= /usr/local -endif - +PREFIX ?= /usr/local DESTDIR ?= LIBDIR ?= $(PREFIX)/lib INCLUDEDIR ?= $(PREFIX)/include diff --git a/programs/Makefile b/programs/Makefile index 07bec03..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -126,15 +126,15 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else PREFIX ?= /usr/local -endif - DESTDIR ?= BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else MANDIR ?= $(PREFIX)/share/man/man1 +endif INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_SCRIPT ?= $(INSTALL) -m 755 -- cgit v0.12 From 7a73c5c1fe1cf3c95fded86cf5051a70327dcfd1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 11:34:23 +0100 Subject: changed default PREFIX and MANDIR --- lib/Makefile | 7 +------ programs/Makefile | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 3e21c1b..2d9c8f3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,12 +118,7 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else -PREFIX ?= /usr/local -endif - +PREFIX ?= /usr/local DESTDIR ?= LIBDIR ?= $(PREFIX)/lib INCLUDEDIR ?= $(PREFIX)/include diff --git a/programs/Makefile b/programs/Makefile index 07bec03..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -126,15 +126,15 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else PREFIX ?= /usr/local -endif - DESTDIR ?= BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else MANDIR ?= $(PREFIX)/share/man/man1 +endif INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_SCRIPT ?= $(INSTALL) -m 755 -- cgit v0.12 From c8b31263ccbf687c4ea05f29da2011462d1e8e5b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 13:08:38 +0100 Subject: added FUZ_CLEVEL_DEFAULT --- lib/lz4hc.c | 2 +- tests/fuzzer.c | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index ae245b5..b2930de 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -564,7 +564,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; return dictSize; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ff02e0c..dbc15ec 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -63,6 +63,7 @@ typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which do #define FUZ_MAX_BLOCK_SIZE (1 << 17) #define FUZ_MAX_DICT_SIZE (1 << 15) #define FUZ_COMPRESSIBILITY_DEFAULT 60 +#define FUZ_CLEVEL_DEFAULT LZ4HC_CLEVEL_OPT_MIN #define PRIME1 2654435761U #define PRIME2 2246822519U #define PRIME3 3266489917U @@ -349,18 +350,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); /* Test compression */ @@ -466,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -491,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -592,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); + LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1)); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); -- cgit v0.12 From 12aae846b3006d7cd76a1197992eab1a0e995466 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 13:19:11 +0100 Subject: fixed table update in LZ4_loadDictHC --- lib/lz4hc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b2930de..3097df2 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -564,8 +564,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } -- cgit v0.12 From 98f9d6c726605e8138ef3400f46fc845e30c995f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 14:04:38 +0100 Subject: improved logging --- lib/lz4opt.h | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 46797ef..1633453 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -33,6 +33,7 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ +//#include #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) @@ -205,7 +206,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -222,7 +223,7 @@ static int LZ4HC_compress_optimal ( { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; - const BYTE *inr; + const BYTE *inr = NULL; size_t res, cur, cur2; size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; @@ -245,7 +246,7 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), (int)match_num, (int)last_pos); if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -259,7 +260,7 @@ static int LZ4HC_compress_optimal ( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; - LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); @@ -279,28 +280,28 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur-1].litlen + 1; if (cur != litlen) { price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)(cur - litlen), opt[cur - litlen].price, (int)price, (int)cur, (int)litlen); } else { price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)llen); } } else { litlen = 1; price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen)); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)LZ4HC_literalsPrice(litlen)); } mlen = 1; best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, opt[cur].price); if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-(const BYTE*)source), (int)match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -314,7 +315,7 @@ static int LZ4HC_compress_optimal ( mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; - LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)cur2, matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { @@ -329,7 +330,7 @@ static int LZ4HC_compress_optimal ( price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-(const BYTE*)source), (int)mlen, (int)best_mlen, matches[i].off, (int)price, (int)litlen, (int)(cur - litlen), opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } @@ -344,10 +345,10 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off); opt[0].mlen = 1; while (1) { @@ -362,13 +363,13 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ } for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -380,7 +381,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ if (res) return 0; - LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); + LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-(const BYTE*)source), (int)offset); } } -- cgit v0.12 From 1c80b9af4edf94f93582d40bae0856d13eacbf11 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 15:18:19 +0100 Subject: LZ4HC_getSearchNum --- lib/lz4hc.c | 16 ++++++++++++++-- lib/lz4opt.h | 18 +++++++++++------- tests/fuzzer.c | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 3097df2..8ca5442 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -486,6 +486,14 @@ _Search3: return (int) (((char*)op)-dest); } +static int LZ4HC_getSearchNum(int compressionLevel) +{ + switch (compressionLevel) { + default: return 0; /* unused */ + case 11: return 128; + case 12: return 1<<10; + } +} static int LZ4HC_compress_generic ( LZ4HC_CCtx_internal* const ctx, @@ -497,13 +505,14 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { + // printf("LZ4HC_compress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); + case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); + case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -554,11 +563,13 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; + LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + // printf("LZ4_loadDictHC dictSize=%d\n", (int)dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -577,6 +588,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { + // printf("LZ4HC_setExternalDict\n"); if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 1633453..041df52 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -37,6 +37,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_TREE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_OPT_NUM (1<<12) @@ -106,6 +107,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; + LZ4_LOG_TREE("LZ4HC_BinTree_InsertAndGetAllMatches nbAttempts=%d\n", nbAttempts); + if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -117,6 +120,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); + LZ4_LOG_TREE("matchIndex[%u] current[%u] lowLimit[%u]\n", matchIndex, current, lowLimit); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { @@ -141,29 +145,30 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matches[mnum].len = (int)matchLength; mnum++; } - if (best_mlen > LZ4_OPT_NUM) break; + if (best_mlen > LZ4_OPT_NUM) { LZ4_LOG_TREE("best_mlen > LZ4_OPT_NUM\n"); break; } } if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + { LZ4_LOG_TREE("ip+matchLength > iHighLimit\n"); break; } /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) break; + if (*ptr0 == (U16)-1) { LZ4_LOG_TREE("*ptr0 == (U16)-1"); break; } delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) break; + if (*ptr1 == (U16)-1) { LZ4_LOG_TREE("*ptr1 == (U16)-1\n"); break; } delta1 = *ptr1; delta0 += delta1; matchIndex -= delta1; } } + LZ4_LOG_TREE("nbAttempts=%d mnum=%d\n", nbAttempts, mnum); *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; @@ -178,7 +183,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdateBT; - + LZ4_LOG_TREE("LZ4HC_updateBinTree %d->%d nbAttempts=%d\n", idx, target, ctx->searchNum); while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -206,7 +211,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + LZ4_LOG_PARSER("%u: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -330,7 +335,6 @@ static int LZ4HC_compress_optimal ( price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-(const BYTE*)source), (int)mlen, (int)best_mlen, matches[i].off, (int)price, (int)litlen, (int)(cur - litlen), opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index dbc15ec..fb918b4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -601,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); -- cgit v0.12 From 3d5bb38977cbb200fe0950095b7a10261b7ff792 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 15:37:12 +0100 Subject: clean logging --- lib/lz4hc.c | 3 --- lib/lz4opt.h | 49 ++++--------------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8ca5442..45dea72 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -505,7 +505,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - // printf("LZ4HC_compress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { @@ -569,7 +568,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - // printf("LZ4_loadDictHC dictSize=%d\n", (int)dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -588,7 +586,6 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - // printf("LZ4HC_setExternalDict\n"); if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 041df52..cc80495 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -33,13 +33,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -//#include -#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_TREE(fmt, ...) //printf(fmt, __VA_ARGS__) - - #define LZ4_OPT_NUM (1<<12) @@ -107,8 +100,6 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; - LZ4_LOG_TREE("LZ4HC_BinTree_InsertAndGetAllMatches nbAttempts=%d\n", nbAttempts); - if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -120,7 +111,6 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); - LZ4_LOG_TREE("matchIndex[%u] current[%u] lowLimit[%u]\n", matchIndex, current, lowLimit); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { @@ -145,30 +135,29 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matches[mnum].len = (int)matchLength; mnum++; } - if (best_mlen > LZ4_OPT_NUM) { LZ4_LOG_TREE("best_mlen > LZ4_OPT_NUM\n"); break; } + if (best_mlen > LZ4_OPT_NUM) break; } if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - { LZ4_LOG_TREE("ip+matchLength > iHighLimit\n"); break; } /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) { LZ4_LOG_TREE("*ptr0 == (U16)-1"); break; } + if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) { LZ4_LOG_TREE("*ptr1 == (U16)-1\n"); break; } + if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; matchIndex -= delta1; } } - LZ4_LOG_TREE("nbAttempts=%d mnum=%d\n", nbAttempts, mnum); *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; @@ -183,7 +172,6 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdateBT; - LZ4_LOG_TREE("LZ4HC_updateBinTree %d->%d nbAttempts=%d\n", idx, target, ctx->searchNum); while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -211,7 +199,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%u: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -251,7 +238,6 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), (int)match_num, (int)last_pos); if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -265,7 +251,6 @@ static int LZ4HC_compress_optimal ( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; - LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); @@ -285,29 +270,22 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur-1].litlen + 1; if (cur != litlen) { price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)(cur - litlen), opt[cur - litlen].price, (int)price, (int)cur, (int)litlen); } else { price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)llen); } } else { litlen = 1; price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)LZ4HC_literalsPrice(litlen)); } mlen = 1; best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, opt[cur].price); if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-(const BYTE*)source), (int)match_num); - if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -320,7 +298,6 @@ static int LZ4HC_compress_optimal ( mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; - LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)cur2, matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { @@ -348,12 +325,6 @@ static int LZ4HC_compress_optimal ( cur = last_pos - best_mlen; encode: /* cur, last_pos, best_mlen, best_off have to be set */ - for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); - } - - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off); - opt[0].mlen = 1; while (1) { mlen = opt[cur].mlen; @@ -366,26 +337,15 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ cur -= mlen; } - for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); - i += opt[i].mlen; - } - cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen); res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); - LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); - if (res) return 0; - - LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-(const BYTE*)source), (int)offset); } } @@ -394,7 +354,6 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Wed, 28 Dec 2016 15:38:59 +0100 Subject: removed nextToUpdateBT --- lib/lz4hc.c | 5 ++--- lib/lz4hc.h | 2 -- lib/lz4opt.h | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 45dea72..5d4ea3e 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB; + hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; @@ -597,7 +597,7 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */ + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, @@ -657,7 +657,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; - if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit; } return dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index ad731d9..1036fd0 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -154,7 +154,6 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ - uint32_t nextToUpdateBT; /* index from which to continue binary tree update */ uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -172,7 +171,6 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - unsigned int nextToUpdateBT; /* index from which to continue binary tree update */ unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; diff --git a/lib/lz4opt.h b/lib/lz4opt.h index cc80495..d1913fe 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -171,7 +171,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdateBT; + U32 idx = ctx->nextToUpdate; while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -184,10 +184,10 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) { int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ + if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen); + ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); return mnum; } -- cgit v0.12 From d2b51c22d300ff67ef3cd91b905a4c35ff876515 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 17:47:10 +0100 Subject: fuzzer: tests more compression levels --- tests/fuzzer.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fb918b4..b129c96 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -63,7 +63,6 @@ typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which do #define FUZ_MAX_BLOCK_SIZE (1 << 17) #define FUZ_MAX_DICT_SIZE (1 << 15) #define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define FUZ_CLEVEL_DEFAULT LZ4HC_CLEVEL_OPT_MIN #define PRIME1 2654435761U #define PRIME2 2246822519U #define PRIME3 3266489917U @@ -304,6 +303,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart); + int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); char* const block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; @@ -350,13 +350,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ @@ -467,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -492,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -593,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1)); + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); @@ -656,7 +656,7 @@ _output_error: #define testCompressedSize (128 KB) #define ringBufferSize (8 KB) -static void FUZ_unitTests(void) +static void FUZ_unitTests(int compressionLevel) { const unsigned testNb = 0; const unsigned seed = 0; @@ -751,7 +751,7 @@ static void FUZ_unitTests(void) /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); @@ -762,7 +762,7 @@ static void FUZ_unitTests(void) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); @@ -776,7 +776,7 @@ static void FUZ_unitTests(void) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); @@ -793,7 +793,7 @@ static void FUZ_unitTests(void) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); @@ -814,7 +814,7 @@ static void FUZ_unitTests(void) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -860,7 +860,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); while (iNext + messageSize < testCompressedSize) { @@ -902,7 +902,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); #define BSIZE1 65537 @@ -957,7 +957,7 @@ static void FUZ_unitTests(void) } } - printf("All unit tests completed successfully \n"); + printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel); return; _output_error: exit(1); @@ -1103,7 +1103,7 @@ int main(int argc, const char** argv) if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); + if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } if (nbTests<=0) nbTests=1; -- cgit v0.12 From 9e867db90adeacd3f2e241c1f3d1097640c69c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 10:14:35 +0100 Subject: cmake: Fix SOVERSION to match Makefiles Fix SOVERSION to use only major lz4 version, as Makefiles do. This ensure that CMake uses 'liblz.so.1' SONAME and creates 'liblz.so.1' symlink. --- contrib/cmake_unofficial/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index de070d6..2380a0b 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -99,7 +99,7 @@ endif() # liblz4 add_library(lz4 ${LZ4_SOURCES}) set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_STRING}" + SOVERSION "${LZ4_VERSION_MAJOR}" VERSION "${LZ4_VERSION_STRING}" POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) -- cgit v0.12 From 28db4acc90805c8bcc183b788900ac566b0346d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 10:34:12 +0100 Subject: cmake: Support building shared & static libs simultaneously Add an additional BUILD_STATIC_LIBS option to control building static libraries independently of shared. This makes it possible (if both options are set to ON) to build both shared and static libraries simulataneously. A dependant option is used to preserve the current BUILD_SHARED_LIBS behavior, i.e. -DBUILD_SHARED_LIBS=ON -- shared lib only, -DBUILD_SHARED_LIBS=OFF -- static lib only. The targets used to build shared and static library are split now, and only relevant properties are passed to each of them. An alias is used to link programs to the preferred library. --- contrib/cmake_unofficial/CMakeLists.txt | 44 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index 2380a0b..df16d46 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -65,6 +65,11 @@ endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED) # which case we always use static libraries. include(CMakeDependentOption) CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF) +CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON) + +if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS) + message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled") +endif() set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib") set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs") @@ -90,28 +95,41 @@ set(LZ4_CLI_SOURCES # we're building a shared library this is ignored and PIC is always # used. option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON) -if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS) - set(LZ4_POSITION_INDEPENDENT_CODE TRUE) -else() - set(LZ4_POSITION_INDEPENDENT_CODE FALSE) -endif() # liblz4 -add_library(lz4 ${LZ4_SOURCES}) -set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_MAJOR}" - VERSION "${LZ4_VERSION_STRING}" - POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) +set(LZ4_LIBRARIES_BUILT) +if(BUILD_SHARED_LIBS) + add_library(lz4_shared SHARED ${LZ4_SOURCES}) + set_target_properties(lz4_shared PROPERTIES + OUTPUT_NAME lz4 + SOVERSION "${LZ4_VERSION_MAJOR}" + VERSION "${LZ4_VERSION_STRING}") + list(APPEND LZ4_LIBRARIES_BUILT lz4_shared) +endif() +if(BUILD_STATIC_LIBS) + add_library(lz4_static STATIC ${LZ4_SOURCES}) + set_target_properties(lz4_static PROPERTIES + OUTPUT_NAME lz4 + POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB}) + list(APPEND LZ4_LIBRARIES_BUILT lz4_static) +endif() + +# link to shared whenever possible, to static otherwise +if(BUILD_SHARED_LIBS) + set(LZ4_LINK_LIBRARY lz4_shared) +else() + set(LZ4_LINK_LIBRARY lz4_static) +endif() # lz4 add_executable(lz4cli ${LZ4_CLI_SOURCES}) set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4) -target_link_libraries(lz4cli lz4) +target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY}) # lz4c add_executable(lz4c ${LZ4_CLI_SOURCES}) set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS") -target_link_libraries(lz4c lz4) +target_link_libraries(lz4c ${LZ4_LINK_LIBRARY}) # Extra warning flags include (CheckCCompilerFlag) @@ -149,7 +167,7 @@ if(NOT LZ4_BUNDLED_MODE) install(TARGETS lz4cli lz4c RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") - install(TARGETS lz4 + install(TARGETS ${LZ4_LIBRARIES_BUILT} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") install(FILES -- cgit v0.12 From 1380c33b7349507e1d51b505a2f4ee8ab126c587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 15:33:15 +0100 Subject: cmake: Install lz4cat and unlz4 symlinks --- contrib/cmake_unofficial/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index df16d46..38f3b00 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -177,6 +177,18 @@ if(NOT LZ4_BUNDLED_MODE) DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") + + # install lz4cat and unlz4 symlinks on *nix + if(UNIX) + install(CODE " + foreach(f lz4cat unlz4) + set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\") + message(STATUS \"Symlinking: \${dest} -> lz4\") + execute_process( + COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") + endforeach() + ") + endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) # pkg-config -- cgit v0.12 From d7969e49af22943a3f0062b8fe8abc04bb1dd6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 15:57:49 +0100 Subject: cmake: Install manpages --- contrib/cmake_unofficial/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index 38f3b00..9a0983d 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -175,6 +175,8 @@ if(NOT LZ4_BUNDLED_MODE) "${LZ4_LIB_SOURCE_DIR}/lz4frame.h" "${LZ4_LIB_SOURCE_DIR}/lz4hc.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") @@ -188,6 +190,13 @@ if(NOT LZ4_BUNDLED_MODE) COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") endforeach() ") + + # create manpage aliases + foreach(f lz4cat unlz4) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") + endforeach() endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) -- cgit v0.12 From 05e27ade68e8b0db8f0965d218199b024e6fb217 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 3 Jan 2017 03:59:27 +0100 Subject: updated NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index d149128..1d42954 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,11 @@ v1.7.5 -lz4hc : new high compression mode : levels 10-12 compress more and slower, by @inikep. +lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) build : Makefile : fix make -jX lib+exe concurrency (#277) +build : cmake : improvements by Michał Górny (#296) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From 9683a1ae874b2928490d675b66fb80e12c111990 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 5 Jan 2017 16:50:37 +0100 Subject: LZ4_MEMORY_USAGE can be modified from compilation command line --- NEWS | 3 +++ lib/lz4.h | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1d42954..8075947 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v1.7.6 +build : LZ4_MEMORY_USAGE can be modified at compile time, through external define + v1.7.5 lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) diff --git a/lib/lz4.h b/lib/lz4.h index 0aae19c..3dc115b 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -108,8 +108,9 @@ LZ4LIB_API const char* LZ4_versionString (void); * Reduced memory usage can improve speed, due to cache effect * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define LZ4_MEMORY_USAGE 14 - +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE 14 +#endif /*-************************************ * Simple Functions -- cgit v0.12 From 44f95e92ea74d4ee9d56d9d1c524c0a0c9b904d5 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Fri, 13 Jan 2017 00:36:24 +0800 Subject: Fix printf specifier --- programs/lz4cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index cf91a99..0578bde 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -510,7 +510,7 @@ int main(int argc, const char** argv) #ifdef _FILE_OFFSET_BITS DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); #endif - if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); + if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { input_filename = inFileNames[0]; -- cgit v0.12 From db6f733ecc99f5bd08e9e2b1348387d7144d0424 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Fri, 13 Jan 2017 00:43:25 +0800 Subject: Use logical or instead of bitwise or --- programs/bench.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/bench.c b/programs/bench.c index 8311503..8625802 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -231,7 +231,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_getTime(&coolTime); DISPLAYLEVEL(2, "\r%79s\r", ""); - while (!cCompleted | !dCompleted) { + while (!cCompleted || !dCompleted) { UTIL_time_t clockStart; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; -- cgit v0.12 From 489d1618c98903817d516445c18f7c239d07ddbf Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 19 Jan 2017 16:28:08 +0100 Subject: added "This Makefile is validated for" --- Makefile | 2 ++ lib/Makefile | 2 ++ programs/Makefile | 2 ++ tests/Makefile | 2 ++ 4 files changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 1432e6b..efcf28f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ # Copyright (C) Yann Collet 2011-2016 # All rights reserved. # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # BSD license # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: diff --git a/lib/Makefile b/lib/Makefile index 2d9c8f3..7ba9ccb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -3,6 +3,8 @@ # Copyright (C) Yann Collet 2011-2016 # All rights reserved. # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # BSD license # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: diff --git a/programs/Makefile b/programs/Makefile index 060ce21..8a271c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -2,6 +2,8 @@ # LZ4 programs - Makefile # Copyright (C) Yann Collet 2011-2016 # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # GPL v2 License # # This program is free software; you can redistribute it and/or modify diff --git a/tests/Makefile b/tests/Makefile index 97fa782..5f8efb7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,6 +2,8 @@ # LZ4 programs - Makefile # Copyright (C) Yann Collet 2011-2016 # +# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets +# # GPL v2 License # # This program is free software; you can redistribute it and/or modify -- cgit v0.12 From 128244371bb54e0ff9559b7109331a8f2868631d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 23 Jan 2017 15:42:58 +0100 Subject: improved gen_manual --- contrib/gen_manual/gen_manual.cpp | 50 +++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp index 2df081d..9aece63 100644 --- a/contrib/gen_manual/gen_manual.cpp +++ b/contrib/gen_manual/gen_manual.cpp @@ -158,36 +158,28 @@ int main(int argc, char *argv[]) { continue; } - /* comments of type /*= and /**= mean: use a

header and show also all functions until first empty line */ - if ((line.substr(0,3) == "/*=" || line.substr(0,4) == "/**=") && line.find("*/")!=string::npos) { - trim_comments(line); - trim(line, "= "); - sout << "

" << line << "

";
-            lines = get_lines(input, ++linenum, "");
-            for (l=0; l

" << endl; - continue; + spos = line.find("/**="); + if (spos==string::npos) { + spos = line.find("/*!"); + if (spos==string::npos) + spos = line.find("/**"); + if (spos==string::npos) + spos = line.find("/*-"); + if (spos==string::npos) + spos = line.find("/*="); + if (spos==string::npos) + continue; + exclam = line[spos+2]; } + else exclam = '='; - spos = line.find("/*!"); - if (spos==string::npos) - spos = line.find("/**"); - if (spos==string::npos) - spos = line.find("/*-"); - - if (spos==string::npos) - continue; - - exclam = line[spos+2]; comments = get_lines(input, linenum, "*/"); if (!comments.empty()) comments[0] = line.substr(spos+3); if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); for (l=0; l
" << endl << endl; + } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a

header and show also all functions until first empty line */ + trim(comments[0], " "); + sout << "

" << comments[0] << "

";
+            for (l=1; l
";
+            lines = get_lines(input, ++linenum, "");
+            for (l=0; l
" << endl;
         } else { /* comments of type /** and /*- mean: this is a comment; use a 
 header for the first line */
             if (comments.empty()) continue;
 
@@ -238,4 +242,4 @@ int main(int argc, char *argv[]) {
     ostream << "" << endl << "" << endl;
 
     return 0;
-}
+}
\ No newline at end of file
-- 
cgit v0.12


From f9f48f8ed914a8279e24aa81a495ab0f38077d20 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 16:02:51 +0100
Subject: lz4.h: improved manual generation

---
 lib/lz4.h | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 3dc115b..bebe761 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -82,7 +82,7 @@ extern "C" {
 #endif
 
 
-/*========== Version =========== */
+/*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    7    /* for new (non-breaking) interface capabilities */
 #define LZ4_VERSION_RELEASE  5    /* for tweaks, bug-fixes, or development */
@@ -94,8 +94,8 @@ extern "C" {
 #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
 #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
 
-LZ4LIB_API int LZ4_versionNumber (void);
-LZ4LIB_API const char* LZ4_versionString (void);
+LZ4LIB_API int LZ4_versionNumber (void);  /**< library version number; to be used when checking dll version */
+LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string; to be used when checking dll version */
 
 
 /*-************************************
@@ -275,7 +275,8 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict
 ************************************************/
 typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* incomplete type (defined later) */
 
-/* creation / destruction of streaming decompression tracking structure */
+/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
+ *  creation / destruction of streaming decompression tracking structure */
 LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
@@ -401,11 +402,12 @@ union LZ4_streamDecode_u {
 } ;   /* previously typedef'd to LZ4_streamDecode_t */
 
 
-/*=************************************
+/*-************************************
 *  Obsolete Functions
 **************************************/
-/* Deprecation warnings */
-/* Should these warnings be a problem,
+
+/*! Deprecation warnings
+   Should deprecation warnings be a problem,
    it is generally possible to disable them,
    typically with -Wno-deprecated-declarations for gcc
    or _CRT_SECURE_NO_WARNINGS in Visual.
-- 
cgit v0.12


From 6a5633e99cb6e3d6e0a33742a952058bc5238cb4 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 16:03:40 +0100
Subject: lz4 manual updated to v1.7.5

---
 doc/lz4_manual.html | 215 ++++++++++++++++++++++++++++++----------------------
 1 file changed, 126 insertions(+), 89 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index bc46645..ff6e149 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,20 +1,22 @@
 
 
 
-lz4 1.7.2 Manual
+lz4 1.7.5 Manual
 
 
-
lz4 1.7.2 Manual

+
lz4 1.7.5 Manual

 

 
Contents

 
    
 
  1. Introduction
    2. 
-
  2. Tuning parameter
    3. 
-
  3. Private definitions
    4. 
+
  4. Version
    5. 
+
  5. Tuning parameter
    6. 
 
  6. Simple Functions
    7. 
 
  7. Advanced Functions
    8. 
 
  8. Streaming Compression Functions
    9. 
 
  9. Streaming Decompression Functions
    10. 
+
  10. Private definitions
    11. 
+
  11. Obsolete Functions
    12. 
 

 

 
Introduction
@@ -29,19 +31,26 @@
     - unbounded multiple steps (described as Streaming compression)
 
   lz4.h provides block compression functions. It gives full buffer control to user.
-  Block compression functions are not-enough to send information,
-  since it's still necessary to provide metadata (such as compressed size),
-  and each application can do it in whichever way it wants.
-  For interoperability, there is LZ4 frame specification (doc/lz4_Frame_format.md).
+  Decompressing an lz4-compressed block also requires metadata (such as compressed size).
+  Each application is free to encode such metadata in whichever way it wants.
+
+  An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md),
+  take care of encoding standard metadata alongside LZ4-compressed blocks.
+  If your application requires interoperability, it's recommended to use it.
   A library is provided to take care of it, see lz4frame.h.
 

 
-
Version
int LZ4_versionNumber (void);
-const char* LZ4_versionString (void);
+
Version

+
+
int LZ4_versionNumber (void);  /**< library version number; to be used when checking dll version */
 


-
Tuning parameter

+
const char* LZ4_versionString (void);   /**< library version string; to be used when checking dll version */
+


+
Tuning parameter

 
-
#define LZ4_MEMORY_USAGE 14
+
#ifndef LZ4_MEMORY_USAGE
+# define LZ4_MEMORY_USAGE 14
+#endif
 
 Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
  Increasing memory usage improves compression ratio
  Reduced memory usage can improve speed, due to cache effect
@@ -49,44 +58,6 @@ const char* LZ4_versionString (void);
  
 


 
-
Private definitions
- Do not use these definitions.
- They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
- If you use these definitions in your code, it will break when you upgrade LZ4 to a new version.
-

-
-
typedef struct {
-    uint32_t hashTable[LZ4_HASH_SIZE_U32];
-    uint32_t currentOffset;
-    uint32_t initCheck;
-    const uint8_t* dictionary;
-    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
-    uint32_t dictSize;
-} LZ4_stream_t_internal;
-


-
typedef struct {
-    const uint8_t* externalDict;
-    size_t extDictSize;
-    const uint8_t* prefixEnd;
-    size_t prefixSize;
-} LZ4_streamDecode_t_internal;
-


-
typedef struct {
-    unsigned int hashTable[LZ4_HASH_SIZE_U32];
-    unsigned int currentOffset;
-    unsigned int initCheck;
-    const unsigned char* dictionary;
-    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
-    unsigned int dictSize;
-} LZ4_stream_t_internal;
-


-
typedef struct {
-    const unsigned char* externalDict;
-    size_t extDictSize;
-    const unsigned char* prefixEnd;
-    size_t prefixSize;
-} LZ4_streamDecode_t_internal;
-


 
Simple Functions

 
 
int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
@@ -178,30 +149,16 @@ int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int
 
 
Streaming Compression Functions

 
-
typedef struct {
-  union {
-    long long table[LZ4_STREAMSIZE_U64];
-    LZ4_stream_t_internal internal_donotuse;
-  };
-} LZ4_stream_t;
-
 information structure to track an LZ4 stream.
- important : init this structure content before first use !
- note : only allocated directly the structure if you are statically linking LZ4
-        If you are using liblz4 as a DLL, please use below construction methods instead.
- 
-


-
-
void LZ4_resetStream (LZ4_stream_t* streamPtr);
-
  Use this function to init an allocated `LZ4_stream_t` structure
- 
-


-
 
LZ4_stream_t* LZ4_createStream(void);
 int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
  LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure.
   LZ4_freeStream() releases its memory.
-  In the context of a DLL (liblz4), please use these methods rather than the static struct.
-  They are more future proof, in case of a change of `LZ4_stream_t` size.
+ 
+


+
+
void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
  An LZ4_stream_t structure can be allocated once and re-used multiple times.
+  Use this function to init an allocated `LZ4_stream_t` structure and start a new compression.
  
 


 
@@ -231,24 +188,12 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
  
 


 
-
Streaming Decompression Functions

+
Streaming Decompression Functions
  Bufferless synchronous API
+

 
-
typedef struct {
-  union {
-    unsigned long long table[LZ4_STREAMDECODESIZE_U64];
-    LZ4_streamDecode_t_internal internal_donotuse;
-  };
-


 
LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-
 information structure to track an LZ4 stream.
- init this structure content using LZ4_setStreamDecode or memset() before first use !
-
- In the context of a DLL (liblz4) please prefer usage of construction methods below.
- They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
- LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
- LZ4_freeStreamDecode releases its memory.
- 
+
  creation / destruction of streaming decompression tracking structure 
 


 
 
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
@@ -278,10 +223,102 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
 
 
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
 int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
-
Advanced decoding functions :
-    These decoding functions work the same as
-    a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
-    They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
+
  These decoding functions work the same as
+  a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
+  They are stand-alone, and don't need an LZ4_streamDecode_t structure.
+ 
+


+
+
Private definitions
+ Do not use these definitions.
+ They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
+ Using these definitions will expose code to API and/or ABI break in future versions of the library.
+

+
+
typedef struct {
+    uint32_t hashTable[LZ4_HASH_SIZE_U32];
+    uint32_t currentOffset;
+    uint32_t initCheck;
+    const uint8_t* dictionary;
+    uint8_t* bufferStart;   /* obsolete, used for slideInputBuffer */
+    uint32_t dictSize;
+} LZ4_stream_t_internal;
+


+
typedef struct {
+    const uint8_t* externalDict;
+    size_t extDictSize;
+    const uint8_t* prefixEnd;
+    size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+


+
typedef struct {
+    unsigned int hashTable[LZ4_HASH_SIZE_U32];
+    unsigned int currentOffset;
+    unsigned int initCheck;
+    const unsigned char* dictionary;
+    unsigned char* bufferStart;   /* obsolete, used for slideInputBuffer */
+    unsigned int dictSize;
+} LZ4_stream_t_internal;
+


+
typedef struct {
+    const unsigned char* externalDict;
+    size_t extDictSize;
+    const unsigned char* prefixEnd;
+    size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+


+
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
+union LZ4_stream_u {
+    unsigned long long table[LZ4_STREAMSIZE_U64];
+    LZ4_stream_t_internal internal_donotuse;
+} ;  /* previously typedef'd to LZ4_stream_t */
+
 information structure to track an LZ4 stream.
+ init this structure before first use.
+ note : only use in association with static linking !
+        this definition is not API/ABI safe,
+        and may change in a future version !
+ 
+


+
+
#define LZ4_STREAMDECODESIZE_U64  4
+#define LZ4_STREAMDECODESIZE     (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+union LZ4_streamDecode_u {
+    unsigned long long table[LZ4_STREAMDECODESIZE_U64];
+    LZ4_streamDecode_t_internal internal_donotuse;
+} ;   /* previously typedef'd to LZ4_streamDecode_t */
+
 information structure to track an LZ4 stream during decompression.
+ init this structure  using LZ4_setStreamDecode (or memset()) before first use
+ note : only use in association with static linking !
+        this definition is not API/ABI safe,
+        and may change in a future version !
+ 
+


+
+
Obsolete Functions

+
+
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
+#  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
+#else
+#  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#    define LZ4_DEPRECATED(message) [[deprecated(message)]]
+#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ4_GCC_VERSION >= 301)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ4_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
+#    define LZ4_DEPRECATED(message)
+#  endif
+#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
+
   Should deprecation warnings be a problem,
+   it is generally possible to disable them,
+   typically with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual.
+   Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS 
 


 
 
-- 
cgit v0.12


From 9071df0fa54ffad456dae41fe4463587cb6eeb4f Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 16:22:00 +0100
Subject: lz4frame.h: prepared to generate manual

---
 lib/lz4frame.h | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index a4a4ce6..02852d4 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -48,10 +48,10 @@ extern "C" {
 /* ---   Dependency   --- */
 #include    /* size_t */
 
-/*-***************************************************************
+/*^***************************************************************
 *  Compiler specifics
 *****************************************************************/
-/*!
+/*
 *  LZ4_DLL_EXPORT :
 *  Enable exporting of functions when building a Windows DLL
 */
@@ -77,8 +77,8 @@ extern "C" {
 **************************************/
 typedef size_t LZ4F_errorCode_t;
 
-LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);
-LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /* return error code string; useful for debugging */
+LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
+LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
 
 
 /*-************************************
@@ -130,7 +130,7 @@ typedef LZ4F_frameType_t frameType_t;
 typedef LZ4F_contentChecksum_t contentChecksum_t;
 #endif
 
-/* LZ4F_frameInfo_t :
+/*! LZ4F_frameInfo_t :
  * makes it possible to supply detailed frame parameters to the stream interface.
  * It's not required to set all fields, as long as the structure was initially memset() to zero.
  * All reserved fields must be set to zero. */
@@ -143,7 +143,7 @@ typedef struct {
   unsigned               reserved[2];           /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
-/* LZ4F_preferences_t :
+/*! LZ4F_preferences_t :
  * makes it possible to supply detailed compression parameters to the stream interface.
  * It's not required to set all fields, as long as the structure was initially memset() to zero.
  * All reserved fields must be set to zero. */
@@ -192,9 +192,7 @@ typedef struct {
 
 #define LZ4F_VERSION 100
 LZ4FLIB_API unsigned LZ4F_getVersion(void);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
-/* LZ4F_createCompressionContext() :
+/*! LZ4F_createCompressionContext() :
  * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
  * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
  * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
@@ -202,30 +200,31 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  * If @return != zero, there was an error during context creation.
  * Object can release its memory using LZ4F_freeCompressionContext();
  */
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 
 /* Compression */
 
 #define LZ4F_HEADER_SIZE_MAX 15
-LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
-/* LZ4F_compressBegin() :
+/*! LZ4F_compressBegin() :
  * will write the frame header into dstBuffer.
  * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
  * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
  * @return : number of bytes written into dstBuffer for the header
  *           or an error code (which can be tested using LZ4F_isError())
  */
+LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
 
-LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-/* LZ4F_compressBound() :
+/*! LZ4F_compressBound() :
  * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
  * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
  * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  */
+LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
 
-LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_compressUpdate() :
+/*! LZ4F_compressUpdate() :
  * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
  * This value is provided by LZ4F_compressBound().
@@ -235,9 +234,9 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
+LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
 
-LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_flush() :
+/*! LZ4F_flush() :
  * When data must be generated and sent immediately, without waiting for a block to be completely filled,
  * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
  * `dstCapacity` must be large enough to ensure the operation will be successful.
@@ -245,9 +244,9 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci
  * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
+LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
 
-LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-/* LZ4F_compressEnd() :
+/*! LZ4F_compressEnd() :
  * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
  * It will flush whatever data remained within `cctx` (like LZ4_flush())
  * and properly finalize the frame, with an endMark and a checksum.
@@ -256,6 +255,7 @@ LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dst
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  */
+LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
 
 
 /*-*********************************
@@ -285,7 +285,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
 
 
-/*======   Decompression   ======*/
+/* ======   Decompression   ======*/
 
 /*!LZ4F_getFrameInfo() :
  * This function decodes frame header information (such as max blockSize, frame checksum, etc.).
-- 
cgit v0.12


From f54c7e0e771e09ca6575148ea5a4bc4dd005e1d3 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 16:31:42 +0100
Subject: added lz4frame_manual.html

---
 doc/lz4frame_manual.html | 240 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 240 insertions(+)
 create mode 100644 doc/lz4frame_manual.html

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
new file mode 100644
index 0000000..f76ec3d
--- /dev/null
+++ b/doc/lz4frame_manual.html
@@ -0,0 +1,240 @@
+
+
+
+lz4frame 1.7.5 Manual
+
+
+
lz4frame 1.7.5 Manual

+

+
Contents

+
    
+
  1. Introduction
    2. 
+
  2. Error management
    3. 
+
  3. Frame compression types
    4. 
+
  4. Simple compression function
    5. 
+
  5. Advanced compression functions
    6. 
+
  6. Decompression functions
    7. 
+

+

+
Introduction
+  lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
+  lz4frame.h provides frame compression functions that take care
+  of encoding standard metadata alongside LZ4-compressed blocks.
+

+
+
Error management

+
+
unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
+


+
const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
+


+
Frame compression types

+
+
typedef enum {
+    LZ4F_default=0,
+    LZ4F_max64KB=4,
+    LZ4F_max256KB=5,
+    LZ4F_max1MB=6,
+    LZ4F_max4MB=7
+    LZ4F_OBSOLETE_ENUM(max64KB)
+    LZ4F_OBSOLETE_ENUM(max256KB)
+    LZ4F_OBSOLETE_ENUM(max1MB)
+    LZ4F_OBSOLETE_ENUM(max4MB)
+} LZ4F_blockSizeID_t;
+


+
typedef enum {
+    LZ4F_blockLinked=0,
+    LZ4F_blockIndependent
+    LZ4F_OBSOLETE_ENUM(blockLinked)
+    LZ4F_OBSOLETE_ENUM(blockIndependent)
+} LZ4F_blockMode_t;
+


+
typedef enum {
+    LZ4F_noContentChecksum=0,
+    LZ4F_contentChecksumEnabled
+    LZ4F_OBSOLETE_ENUM(noContentChecksum)
+    LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)
+} LZ4F_contentChecksum_t;
+


+
typedef enum {
+    LZ4F_frame=0,
+    LZ4F_skippableFrame
+    LZ4F_OBSOLETE_ENUM(skippableFrame)
+} LZ4F_frameType_t;
+


+
typedef struct {
+  LZ4F_blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  LZ4F_blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
+  unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
+  unsigned               reserved[2];           /* must be zero for forward compatibility */
+} LZ4F_frameInfo_t;
+
 makes it possible to supply detailed frame parameters to the stream interface.
+ It's not required to set all fields, as long as the structure was initially memset() to zero.
+ All reserved fields must be set to zero. 
+


+
+
typedef struct {
+  LZ4F_frameInfo_t frameInfo;
+  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+  unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
+  unsigned reserved[4];            /* must be zero for forward compatibility */
+} LZ4F_preferences_t;
+
 makes it possible to supply detailed compression parameters to the stream interface.
+ It's not required to set all fields, as long as the structure was initially memset() to zero.
+ All reserved fields must be set to zero. 
+


+
+
Simple compression function

+
+
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
+
 Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+ 
+


+
+
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
+
 Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
+ An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
+ This value is supplied by LZ4F_compressFrameBound().
+ If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
+ The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ @return : number of bytes written into dstBuffer.
+           or an error code if it fails (can be tested using LZ4F_isError())
+ 
+


+
+
Advanced compression functions

+
+
typedef struct {
+  unsigned stableSrc;    /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
+  unsigned reserved[3];
+} LZ4F_compressOptions_t;
+


+
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
+LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
+
 The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
+ The function will provide a pointer to a fully allocated LZ4F_cctx object.
+ If @return != zero, there was an error during context creation.
+ Object can release its memory using LZ4F_freeCompressionContext();
+ 
+


+
+
size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
+
 will write the frame header into dstBuffer.
+ dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+ `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
+ @return : number of bytes written into dstBuffer for the header
+           or an error code (which can be tested using LZ4F_isError())
+ 
+


+
+
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
+
 Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
+ prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
+ Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
+ When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+ 
+


+
+
size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+
 LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
+ This value is provided by LZ4F_compressBound().
+ If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
+ LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
+ `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+           or an error code if it fails (which can be tested using LZ4F_isError())
+ 
+


+
+
size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+
 When data must be generated and sent immediately, without waiting for a block to be completely filled,
+ it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
+ `dstCapacity` must be large enough to ensure the operation will be successful.
+ `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
+ @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
+           or an error code if it fails (which can be tested using LZ4F_isError())
+ 
+


+
+
size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
+
 To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+ It will flush whatever data remained within `cctx` (like LZ4_flush())
+ and properly finalize the frame, with an endMark and a checksum.
+ `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
+ @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+           or an error code if it fails (which can be tested using LZ4F_isError())
+ A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+ 
+


+
+
Decompression functions

+
+
typedef struct {
+  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned reserved[3];
+} LZ4F_decompressOptions_t;
+


+
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+
 Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
+ The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
+ The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+ The result is an errorCode, which can be tested using LZ4F_isError().
+ dctx memory can be released using LZ4F_freeDecompressionContext();
+ The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ That is, it should be == 0 if decompression has been completed fully and correctly.
+ 
+


+
+
size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
+                                     LZ4F_frameInfo_t* frameInfoPtr,
+                                     const void* srcBuffer, size_t* srcSizePtr);
+
 This function decodes frame header information (such as max blockSize, frame checksum, etc.).
+ Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
+ A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
+ The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ Decompression must resume from this point (srcBuffer + *srcSizePtr).
+ Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
+ Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
+ @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
+           or an error code which can be tested using LZ4F_isError()
+           (typically, when there is not enough src bytes to fully decode the frame header)
+ 
+


+
+
size_t LZ4F_decompress(LZ4F_dctx* dctx,
+                                   void* dstBuffer, size_t* dstSizePtr,
+                                   const void* srcBuffer, size_t* srcSizePtr,
+                                   const LZ4F_decompressOptions_t* dOptPtr);
+
 Call this function repetitively to regenerate data compressed within `srcBuffer`.
+ The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+
+ The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+
+ The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
+ It typically happens when dstBuffer is not large enough to contain all decoded data.
+ Remaining data will have to be presented again in a subsequent invocation.
+
+ `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
+ `dstBuffer` can be changed at will between each consecutive function invocation.
+
+ @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ This is just a hint though, it's always possible to provide any srcSize.
+ When a frame is fully decoded, @return will be 0 (no more data expected).
+ If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+
+ After a frame is fully decoded, dctx can be used again to decompress another frame.
+ 
+


+
+
+
-- 
cgit v0.12


From d56ee32b394f95e4403ccce4a38fe633e77c752b Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 16:33:03 +0100
Subject: updated gen-lz4-manual.sh

---
 contrib/gen_manual/gen-lz4-manual.sh | 3 ++-
 contrib/gen_manual/gen_manual.cpp    | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/contrib/gen_manual/gen-lz4-manual.sh b/contrib/gen_manual/gen-lz4-manual.sh
index 55d31a4..73a7214 100644
--- a/contrib/gen_manual/gen-lz4-manual.sh
+++ b/contrib/gen_manual/gen-lz4-manual.sh
@@ -6,4 +6,5 @@ LIBVER_PATCH_SCRIPT=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][
 LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
 
 echo LZ4_VERSION=$LIBVER_SCRIPT
-./gen_manual $LIBVER_SCRIPT ../../lib/lz4.h ./lz4_manual.html
+./gen_manual "lz4 $LIBVER_SCRIPT" ../../lib/lz4.h ./lz4_manual.html
+./gen_manual "lz4frame $LIBVER_SCRIPT" ../../lib/lz4frame.h ./lz4frame_manual.html
diff --git a/contrib/gen_manual/gen_manual.cpp b/contrib/gen_manual/gen_manual.cpp
index 9aece63..65abd3a 100644
--- a/contrib/gen_manual/gen_manual.cpp
+++ b/contrib/gen_manual/gen_manual.cpp
@@ -89,11 +89,13 @@ vector get_lines(vector& input, int& linenum, string terminator)
 /* print line with LZ4LIB_API removed and C++ comments not bold */
 void print_line(stringstream &sout, string line)
 {
-    size_t spos;
+    size_t spos, epos;
 
     if (line.substr(0,11) == "LZ4LIB_API ") line = line.substr(11);
+    if (line.substr(0,12) == "LZ4FLIB_API ") line = line.substr(12);
     spos = line.find("/*");
-    if (spos!=string::npos) {
+    epos = line.find("*/");
+    if (spos!=string::npos && epos!=string::npos) {
         sout << line.substr(0, spos);
         sout << "" << line.substr(spos) << "" << endl;
     } else {
@@ -118,7 +120,7 @@ int main(int argc, char *argv[]) {
         return 1;
     }
 
-    version = "lz4 " + string(argv[1]) + " Manual";
+    version = string(argv[1]) + " Manual";
 
     istream.open(argv[2], ifstream::in);
     if (!istream.is_open()) {
-- 
cgit v0.12


From 8e1fd97d16988cb1a894ed46e21493530ee1ddcc Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 23 Jan 2017 17:46:32 +0100
Subject: lz4frame.h: added Introduction

---
 lib/lz4frame.h | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 02852d4..82fbeab 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -48,6 +48,15 @@ extern "C" {
 /* ---   Dependency   --- */
 #include    /* size_t */
 
+
+/**
+  Introduction
+
+  lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md).
+  lz4frame.h provides frame compression functions that take care
+  of encoding standard metadata alongside LZ4-compressed blocks.
+*/
+
 /*^***************************************************************
 *  Compiler specifics
 *****************************************************************/
-- 
cgit v0.12


From 96e150e7df2bd76e3507f64cea61e6d0f72c93ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= 
Date: Sun, 5 Feb 2017 09:45:34 +0100
Subject: [cmake] Fix substituting version in liblz4.pc

Bug: https://bugs.gentoo.org/608144
---
 contrib/cmake_unofficial/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt
index 9a0983d..e59a756 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/contrib/cmake_unofficial/CMakeLists.txt
@@ -215,4 +215,6 @@ else()
   set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
 endif()
 
+# for liblz4.pc substitution
+set(VERSION ${LZ4_VERSION_STRING})
 configure_file(${LZ4_LIB_SOURCE_DIR}/liblz4.pc.in liblz4.pc @ONLY)
-- 
cgit v0.12


From 04ec09269645760604974980bce5b9df13ef830b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 8 Feb 2017 11:02:32 -0800
Subject: updated comments on block sizes

---
 lib/lz4frame.h | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 82fbeab..9d9fe89 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -91,8 +91,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return
 
 
 /*-************************************
-*  Frame compression types
-**************************************/
+ *  Frame compression types
+ **************************************/
 /* #define LZ4F_DISABLE_OBSOLETE_ENUMS */  /* uncomment to disable obsolete enums */
 #ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
 #  define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
@@ -100,6 +100,9 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return
 #  define LZ4F_OBSOLETE_ENUM(x)
 #endif
 
+/* The larger the block size, the (slightly) better the compression ratio,
+ * though there are diminishing returns.
+ * Larger blocks also increase memory usage on both compression and decompression sides. */
 typedef enum {
     LZ4F_default=0,
     LZ4F_max64KB=4,
@@ -112,6 +115,9 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(max4MB)
 } LZ4F_blockSizeID_t;
 
+/* Linked blocks sharply reduce inefficiencies when using small blocks,
+ * they compress better.
+ * However, some LZ4 decoders are only compatible with independent blocks */
 typedef enum {
     LZ4F_blockLinked=0,
     LZ4F_blockIndependent
-- 
cgit v0.12


From 04e5eaf9bc21a1fb5ed8bfcd03ad127bdf5ff4ca Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 9 Feb 2017 03:19:15 -0800
Subject: added DragonFly to list of supported `make install` OS

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index efcf28f..2d0a9d4 100644
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,7 @@ clean:
 #------------------------------------------------------------------------
 #make install is validated only for Linux, OSX, kFreeBSD, Hurd and
 #FreeBSD targets
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly))
 HOST_OS = POSIX
 
 install:
-- 
cgit v0.12


From c7b14967ab5c677086aa1427af4c000fa341343c Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 10 Feb 2017 14:53:58 +0100
Subject: updated platform.h

---
 programs/Makefile   | 3 +--
 programs/platform.h | 6 ++++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 8a271c2..4a8103c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -44,6 +44,7 @@ LZ4DIR  := ../lib
 
 SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)
 OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
+VOID := /dev/null
 
 CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 CFLAGS  ?= -O3
@@ -60,10 +61,8 @@ MD2ROFF_FLAGS  = --roff --warnings --manual="User Commands" --organization="lz4
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
-VOID := nul
 EXT  :=.exe
 else
-VOID := /dev/null
 EXT  :=
 endif
 
diff --git a/programs/platform.h b/programs/platform.h
index f1040c0..f9eb81a 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -59,7 +59,7 @@ extern "C" {
 /* *********************************************************
 *  Turn on Large Files support (>4GB) for 32-bit Linux/Unix
 ***********************************************************/
-#if !defined(__64BIT__)                               /* No point defining Large file for 64 bit */
+#if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW requires it */
 #  if !defined(_FILE_OFFSET_BITS)   
 #    define _FILE_OFFSET_BITS 64                      /* turn off_t into a 64-bit type for ftello, fseeko */
 #  endif
@@ -85,7 +85,9 @@ extern "C" {
 #    define PLATFORM_POSIX_VERSION 200112L
 #  else
 #    if defined(__linux__) || defined(__linux)
-#      define _POSIX_C_SOURCE 200112L  /* use feature test macro */
+#      ifndef _POSIX_C_SOURCE
+#        define _POSIX_C_SOURCE 200112L  /* use feature test macro */
+#      endif
 #    endif
 #    include   /* declares _POSIX_VERSION */
 #    if defined(_POSIX_VERSION)  /* POSIX compliant */
-- 
cgit v0.12


From 5e1a925bea88831b30b750d203c251e63571ccdd Mon Sep 17 00:00:00 2001
From: Benedikt Heine 
Date: Fri, 10 Feb 2017 14:55:54 +0100
Subject: update repolink in makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2d0a9d4..6954562 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 # You can contact the author at :
-#  - LZ4 source repository : https://github.com/Cyan4973/lz4
+#  - LZ4 source repository : https://github.com/lz4/lz4
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
-- 
cgit v0.12


From c139eb40b0d78db390bdc37188eb696d9272f9c4 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 10 Feb 2017 15:00:27 +0100
Subject: added OpenBSD NetBSD SunOS to list of supported `make install` OSes

---
 Makefile       | 8 ++++----
 tests/Makefile | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 2d0a9d4..4229f41 100644
--- a/Makefile
+++ b/Makefile
@@ -86,10 +86,10 @@ clean:
 	@echo Cleaning completed
 
 
-#------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD, Hurd and
-#FreeBSD targets
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly))
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
 
 install:
diff --git a/tests/Makefile b/tests/Makefile
index 5f8efb7..77e6ae7 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -121,9 +121,9 @@ versionsTest:
 	$(PYTHON) test-lz4-versions.py
 
 
-#------------------------------------------------------------------------
-#make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and
-#Solaris targets
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
 MD5:=md5sum
-- 
cgit v0.12


From b89cac7b2e92b792af98bb0a12e4d14684d07629 Mon Sep 17 00:00:00 2001
From: Eric Siegerman 
Date: Tue, 14 Feb 2017 14:17:06 -0500
Subject: Don't use "foo && false || true"

Replace it with either:
    test ! -f $FILE_THAT_SHOULD_NOT_EXIST
or:
    ! $COMMAND_THAT_SHOULD_FAIL

as appropriate.
---
 tests/Makefile | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 77e6ae7..ebab278 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -236,17 +236,17 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
 	@echo "hello world" > tmp
 	$(LZ4) --rm -f tmp
-	ls -ls tmp         && false || true   # must fail (--rm)
-	ls -ls tmp.lz4
-	$(PRGDIR)/lz4cat tmp.lz4              # must display hello world
-	ls -ls tmp.lz4
+	test ! -f tmp                      # must fail (--rm)
+	test   -f tmp.lz4
+	$(PRGDIR)/lz4cat tmp.lz4           # must display hello world
+	test   -f tmp.lz4
 	$(PRGDIR)/unlz4 --rm tmp.lz4
-	ls -ls tmp
-	ls -ls tmp.lz4     && false || true   # must fail (--rm)
-	ls -ls tmp.lz4.lz4 && false || true   # must fail (unlz4)
-	$(PRGDIR)/lz4cat tmp                  # pass-through mode
-	ls -ls tmp
-	ls -ls tmp.lz4     && false || true   # must fail (lz4cat)
+	test   -f tmp
+	test ! -f tmp.lz4                  # must fail (--rm)
+	test ! -f tmp.lz4.lz4              # must fail (unlz4)
+	$(PRGDIR)/lz4cat tmp               # pass-through mode
+	test   -f tmp
+	test ! -f tmp.lz4                  # must fail (lz4cat)
 	$(LZ4) tmp                         # creates tmp.lz4
 	$(PRGDIR)/lz4cat < tmp.lz4 > tmp3  # checks lz4cat works with stdin (#285)
 	$(DIFF) -q tmp tmp3
@@ -262,22 +262,22 @@ test-lz4-hugefile: lz4 datagen
 
 test-lz4-testmode: lz4 datagen
 	@echo "\n ---- bench mode ----"
-	$(LZ4) -bi1
+	  $(LZ4) -bi1
 	@echo "\n ---- test mode ----"
-	./datagen | $(LZ4) -t             && false || true
-	./datagen | $(LZ4) -tf            && false || true
+	! ./datagen | $(LZ4) -t
+	! ./datagen | $(LZ4) -tf
 	@echo "\n ---- pass-through mode ----"
-	./datagen | $(LZ4) -d  > $(VOID)  && false || true
-	./datagen | $(LZ4) -df > $(VOID)
+	! ./datagen | $(LZ4) -d  > $(VOID)
+	  ./datagen | $(LZ4) -df > $(VOID)
 	@echo "Hello World !" > tmp1
 	$(LZ4) -dcf tmp1
 	@echo "from underground..." > tmp2
 	$(LZ4) -dcfm tmp1 tmp2
 	@echo "\n ---- test cli ----"
-	$(LZ4)     file-does-not-exist    && false || true
-	$(LZ4) -f  file-does-not-exist    && false || true
-	$(LZ4) -fm file1-dne file2-dne    && false || true
-	$(LZ4) -fm file1-dne file2-dne    && false || true
+	! $(LZ4)     file-does-not-exist
+	! $(LZ4) -f  file-does-not-exist
+	! $(LZ4) -fm file1-dne file2-dne
+	! $(LZ4) -fm file1-dne file2-dne
 
 test-lz4-opt-parser: lz4 datagen
 	@echo "\n ---- test opt-parser ----"
-- 
cgit v0.12


From eb51b2b8d9254add9d72eb26064518d5bcf1e9ee Mon Sep 17 00:00:00 2001
From: Eric Siegerman 
Date: Tue, 14 Feb 2017 21:58:08 -0500
Subject: Explicitly create $(DESTDIR)$(LIBDIR)/ at install time

This is needed on systems where it isn't the parent of
$(PKGCONFIGDIR), and so doesn't get created implicitly.
---
 lib/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index 7ba9ccb..c4bc7d2 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -143,7 +143,7 @@ liblz4.pc: liblz4.pc.in Makefile
           $< >$@
 
 install: lib liblz4.pc
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ $(DESTDIR)$(LIBDIR)/
 	@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-- 
cgit v0.12


From 40ad1e85d49bd067a0b06b0e923ec47cc9930173 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 15 Feb 2017 17:16:15 +0100
Subject: upgraded util.h

---
 programs/bench.c    |   2 +-
 programs/platform.h |   2 +-
 programs/util.h     | 103 ++++++++++++++++++++++++++++++----------------------
 3 files changed, 62 insertions(+), 45 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 8625802..71d3896 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -393,7 +393,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     if (!pch) pch = strrchr(displayName, '/'); /* Linux */
     if (pch) displayName = pch+1;
 
-    SET_HIGH_PRIORITY;
+    SET_REALTIME_PRIORITY;
 
     if (g_displayLevel == 1 && !g_additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
diff --git a/programs/platform.h b/programs/platform.h
index f9eb81a..51ce1ac 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -59,7 +59,7 @@ extern "C" {
 /* *********************************************************
 *  Turn on Large Files support (>4GB) for 32-bit Linux/Unix
 ***********************************************************/
-#if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW requires it */
+#if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW-w64 requires it */
 #  if !defined(_FILE_OFFSET_BITS)   
 #    define _FILE_OFFSET_BITS 64                      /* turn off_t into a 64-bit type for ftello, fseeko */
 #  endif
diff --git a/programs/util.h b/programs/util.h
index f3ff1b2..044085f 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -70,12 +70,26 @@ extern "C" {
 #endif
 
 
+/* ************************************************************
+* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
+***************************************************************/
+#if defined(_MSC_VER) && (_MSC_VER >= 1400)
+#   define UTIL_fseek _fseeki64
+#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
+#  define UTIL_fseek fseeko
+#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
+#   define UTIL_fseek fseeko64
+#else
+#   define UTIL_fseek fseek
+#endif
+
+
 /*-****************************************
 *  Sleep functions: Windows - Posix - others
 ******************************************/
 #if defined(_WIN32)
 #  include 
-#  define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
+#  define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
 #  define UTIL_sleep(s) Sleep(1000*s)
 #  define UTIL_sleepMilli(milli) Sleep(milli)
 #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
@@ -83,9 +97,9 @@ extern "C" {
 #  include  /* setpriority */
 #  include          /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
 #  if defined(PRIO_PROCESS)
-#    define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
+#    define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
 #  else
-#    define SET_HIGH_PRIORITY /* disabled */
+#    define SET_REALTIME_PRIORITY /* disabled */
 #  endif
 #  define UTIL_sleep(s) sleep(s)
 #  if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L)  /* nanosleep requires POSIX.1-2001 */
@@ -94,7 +108,7 @@ extern "C" {
 #      define UTIL_sleepMilli(milli) /* disabled */
 #  endif
 #else
-#  define SET_HIGH_PRIORITY      /* disabled */
+#  define SET_REALTIME_PRIORITY      /* disabled */
 #  define UTIL_sleep(s)          /* disabled */
 #  define UTIL_sleepMilli(milli) /* disabled */
 #endif
@@ -126,18 +140,26 @@ extern "C" {
 /*-****************************************
 *  Time functions
 ******************************************/
-#if !defined(_WIN32)
-   typedef clock_t UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
-   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-#else
+#if (PLATFORM_POSIX_VERSION >= 1)
+#include 
+#include    /* times */
+   typedef U64 UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
+#elif defined(_WIN32)   /* Windows */
    typedef LARGE_INTEGER UTIL_time_t;
    UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
    UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+#else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
+   typedef clock_t UTIL_time_t;
+   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
+   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
+   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 #endif
 
 
@@ -166,7 +188,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
 ******************************************/
 #if defined(_MSC_VER)
 	#define chmod _chmod
-	typedef struct _stat64 stat_t;
+	typedef struct __stat64 stat_t;
 #else
     typedef struct stat stat_t;
 #endif
@@ -206,63 +228,58 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
 }
 
 
-UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
+UTIL_STATIC int UTIL_isRegFile(const char* infilename)
+{
+    stat_t statbuf;
+    return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
+}
+
+
+UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
 {
     int r;
+    stat_t statbuf;
 #if defined(_MSC_VER)
-    struct _stat64 statbuf;
     r = _stat64(infilename, &statbuf);
-    if (r || !(statbuf.st_mode & S_IFREG)) return 0;   /* No good... */
+    if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
 #else
-    struct stat statbuf;
     r = stat(infilename, &statbuf);
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    if (!r && S_ISDIR(statbuf.st_mode)) return 1;
 #endif
-    return (U64)statbuf.st_size;
-}
-
-
-UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
-{
-    U64 total = 0;
-    unsigned n;
-    for (n=0; n
Date: Wed, 15 Feb 2017 17:17:57 +0100
Subject: Avoid fseek()'s 2GiB barrier with MinGW

---
 programs/lz4io.c | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 640c76d..5264b14 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -59,17 +59,6 @@
 #include "lz4frame.h"
 
 
-/* **************************************
-*  Compiler Options
-****************************************/
-#if defined(_MSC_VER) && (_MSC_VER >= 1400)            /* Avoid MSVC fseek()'s 2GiB barrier */
-#  define fseek _fseeki64
-#endif
-#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
-#  define fseek fseeko
-#endif
-
-
 /*****************************
 *  Constants
 *****************************/
@@ -633,7 +622,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
 
     /* avoid int overflow */
     if (storedSkips > 1 GB) {
-        int const seekResult = fseek(file, 1 GB, SEEK_CUR);
+        int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
         if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
         storedSkips -= 1 GB;
     }
@@ -650,7 +639,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
 
         if (nb0T != seg0SizeT) {   /* not all 0s */
             errno = 0;
-            {   int const seekResult = fseek(file, storedSkips, SEEK_CUR);
+            {   int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
                 if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
             }
             storedSkips = 0;
@@ -670,7 +659,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
         for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
         storedSkips += (unsigned) (restPtr - restStart);
         if (restPtr != restEnd) {
-            int const seekResult = fseek(file, storedSkips, SEEK_CUR);
+            int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
             if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             {   size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
@@ -684,7 +673,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
 static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 {
     if (storedSkips>0) {   /* implies g_sparseFileSupport>0 */
-        int const seekResult = fseek(file, storedSkips-1, SEEK_CUR);
+        int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR);
         if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
         {   const char lastZeroByte[1] = { 0 };
             size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
@@ -873,7 +862,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
     while (offset > 0) {
         unsigned s = offset;
         if (s > stepMax) s = stepMax;
-        errorNb = fseek(fp, (long) s, SEEK_CUR);
+        errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
         if (errorNb != 0) break;
         offset -= s;
     }
-- 
cgit v0.12


From c0a8d0ad8750b2e8e9ac1c8a39b53de765330ccb Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 15 Feb 2017 17:36:00 +0100
Subject: added circle.yml

---
 circle.yml | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 circle.yml

diff --git a/circle.yml b/circle.yml
new file mode 100644
index 0000000..7de1b57
--- /dev/null
+++ b/circle.yml
@@ -0,0 +1,33 @@
+dependencies:
+  override:
+    - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
+    - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
+    - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+    - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 valgrind
+
+test:
+  override:
+  # Tests compilers and C standards
+    - clang -v; make clangtest && make clean
+    - g++ -v; make gpptest     && make clean
+    - gcc -v; make c_standards && make clean
+    - gcc-5 -v; make -C tests test-lz4 clean test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror && make clean
+    - gcc-6 -v; make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
+  # Shorter tests
+    - make cmake               && make clean
+    - make -C lib all          && make clean
+    - make -C tests fullbench-dll && make clean
+    - pyenv global 3.4.4; make versionsTest && make clean
+    - make travis-install      && make clean
+  # Longer tests
+    - make test                && make clean 
+    - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    - make usan                && make clean
+    - clang -v; make staticAnalyze && make clean
+  # Valgrind tests
+    - make -C tests test-mem && make clean
+  # ARM, AArch64, PowerPC, PowerPC64 tests
+    - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static                  && make clean
+    - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean
+    - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static                  && make clean
+    - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static              && make clean
-- 
cgit v0.12


From 4ee9bd5c3ede14f6ac6c31c8e6eb5ffd2fb5c32a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 15 Feb 2017 18:24:42 +0100
Subject: improved Circle CI tests

---
 circle.yml     | 18 ++++++++++++------
 tests/Makefile |  1 +
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/circle.yml b/circle.yml
index 7de1b57..87b4acc 100644
--- a/circle.yml
+++ b/circle.yml
@@ -11,16 +11,22 @@ test:
     - clang -v; make clangtest && make clean
     - g++ -v; make gpptest     && make clean
     - gcc -v; make c_standards && make clean
-    - gcc-5 -v; make -C tests test-lz4 clean test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror && make clean
-    - gcc-6 -v; make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
-  # Shorter tests
+    - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean
+    - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean
+    - gcc-6 -v; make c_standards CC=gcc-6 && make clean
+    - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
+# Shorter tests
     - make cmake               && make clean
-    - make -C lib all          && make clean
-    - make -C tests fullbench-dll && make clean
+    - make test-lz4
+    - make test-lz4c
+    - make test-fasttest
+    - make test-frametest
+    - make test-fullbench
+    - make test-fuzzer && make clean
+    - make -C lib all && make -C tests fullbench-dll && make clean
     - pyenv global 3.4.4; make versionsTest && make clean
     - make travis-install      && make clean
   # Longer tests
-    - make test                && make clean 
     - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
     - make usan                && make clean
     - clang -v; make staticAnalyze && make clean
diff --git a/tests/Makefile b/tests/Makefile
index ebab278..d1117c6 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -301,6 +301,7 @@ test-lz4c: lz4c datagen
 	./datagen -g256MB | $(LZ4)c -l -v    | $(LZ4)c   -t
 
 test-lz4c32: CFLAGS+=-m32
+test-lz4c32: MOREFLAGS+="-I/usr/include/x86_64-linux-gnu"
 test-lz4c32: test-lz4
 
 test-interop-32-64: lz4 lz4c32 datagen
-- 
cgit v0.12


From 0d77c85d9dd7592f23a3f0147d570d3d0180dcd7 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 15 Feb 2017 22:10:11 +0100
Subject: circle.yml: fixed lz4c32 target

---
 circle.yml     | 16 ++++++++--------
 tests/Makefile |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/circle.yml b/circle.yml
index 87b4acc..dafffe8 100644
--- a/circle.yml
+++ b/circle.yml
@@ -3,7 +3,7 @@ dependencies:
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
     - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
     - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 valgrind
+    - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind
 
 test:
   override:
@@ -17,13 +17,13 @@ test:
     - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean
 # Shorter tests
     - make cmake               && make clean
-    - make test-lz4
-    - make test-lz4c
-    - make test-fasttest
-    - make test-frametest
-    - make test-fullbench
-    - make test-fuzzer && make clean
-    - make -C lib all && make -C tests fullbench-dll && make clean
+    - make -C tests test-lz4
+    - make -C tests test-lz4c
+    - make -C tests test-fasttest
+    - make -C tests test-frametest
+    - make -C tests test-fullbench
+    - make -C tests test-fuzzer && make clean
+    - make -C lib all && make clean
     - pyenv global 3.4.4; make versionsTest && make clean
     - make travis-install      && make clean
   # Longer tests
diff --git a/tests/Makefile b/tests/Makefile
index d1117c6..e362626 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -78,7 +78,7 @@ lz4c:
 	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
 
 lz4c32:   # create a 32-bits version for 32/64 interop tests
-	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
+	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS) -I/usr/include/x86_64-linux-gnu"
 	cp $(LZ4) $(LZ4)c32
 
 fullbench  : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
-- 
cgit v0.12


From ace22424d87d63900707fd0645b5dc411fad18b6 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 15 Feb 2017 23:19:30 +0100
Subject: circle.yml: fixed lz4c32 target (2)

---
 programs/Makefile | 2 +-
 tests/Makefile    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 4a8103c..6037355 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -86,7 +86,7 @@ lz4c  : CFLAGS += $(DEBUGFLAGS)
 lz4c  : $(SRCFILES)
 	$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
 
-lz4c32: CFLAGS += -m32 $(DEBUGFLAGS)
+lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) -I/usr/include/x86_64-linux-gnu
 lz4c32: $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
diff --git a/tests/Makefile b/tests/Makefile
index e362626..d1117c6 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -78,7 +78,7 @@ lz4c:
 	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
 
 lz4c32:   # create a 32-bits version for 32/64 interop tests
-	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS) -I/usr/include/x86_64-linux-gnu"
+	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
 	cp $(LZ4) $(LZ4)c32
 
 fullbench  : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
-- 
cgit v0.12


From ca2c5fd3c6e18dd3aaacdf59dbae0c0edfad8b10 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Thu, 16 Feb 2017 00:17:42 +0100
Subject: CFLAGS=-I/usr/include/x86_64-linux-gnu

---
 circle.yml        | 2 +-
 programs/Makefile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/circle.yml b/circle.yml
index dafffe8..b3faae0 100644
--- a/circle.yml
+++ b/circle.yml
@@ -24,7 +24,7 @@ test:
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
     - make -C lib all && make clean
-    - pyenv global 3.4.4; make versionsTest && make clean
+    - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean
     - make travis-install      && make clean
   # Longer tests
     - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
diff --git a/programs/Makefile b/programs/Makefile
index 6037355..4a8103c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -86,7 +86,7 @@ lz4c  : CFLAGS += $(DEBUGFLAGS)
 lz4c  : $(SRCFILES)
 	$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
 
-lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) -I/usr/include/x86_64-linux-gnu
+lz4c32: CFLAGS += -m32 $(DEBUGFLAGS)
 lz4c32: $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-- 
cgit v0.12


From d3eaf788380d430217bd5f4efe2feed30662892a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Thu, 16 Feb 2017 16:30:16 +0100
Subject: remove MOREFLAGS from test-lz4c32

---
 tests/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index d1117c6..ebab278 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -301,7 +301,6 @@ test-lz4c: lz4c datagen
 	./datagen -g256MB | $(LZ4)c -l -v    | $(LZ4)c   -t
 
 test-lz4c32: CFLAGS+=-m32
-test-lz4c32: MOREFLAGS+="-I/usr/include/x86_64-linux-gnu"
 test-lz4c32: test-lz4
 
 test-interop-32-64: lz4 lz4c32 datagen
-- 
cgit v0.12


From 97df1c9789cbc8a7891cadbd49ea5053574e2f72 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 15:12:24 -0800
Subject: updated LICENSE

to avoid risks of confusion on GPLv2 licensed code.

GPLv2 code is not _intended_ to be integrated into 3rd party application,
but it can be used for that nonetheless (provided compliance with GPLv2 licence).
It just receives less focus and support into this direction.
---
 LICENSE | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/LICENSE b/LICENSE
index df1edbc..c221aeb 100644
--- a/LICENSE
+++ b/LICENSE
@@ -3,13 +3,9 @@ This repository uses 2 different licenses :
 - all other files use a GPLv2 license, unless explicitly stated otherwise
 
 Relevant license is reminded at the top of each source file,
-and with the presence of COPYING or LICENSE file.
+and with presence of COPYING or LICENSE file in associated directories.
 
-This model emphasizes the fact that
-only files in the `lib` directory are designed to be included into 3rd party projects.
-
-Other files, such as those from `programs` or `examples` directory,
-are not intended to be compiled outside of their context.
-They can serve as source of inspiration,
-but they should not be copy/pasted into 3rd party projects,
-as this scenario is not supported.
+This model is selected to emphasize that
+files in the `lib` directory are designed to be included into 3rd party applications,
+while all other files, in `programs`, `tests` or `examples`,
+receive more limited attention and support for such scenario.
-- 
cgit v0.12


From 0dfb0b9dad2a8cb7cc347d2139bf9b84de7e1481 Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Sun, 5 Mar 2017 23:20:10 +0000
Subject: Fix test-lz4-basic

When no output filename is specified and stdout is not a terminal,
lz4 doesn't attempt to guess an output filename and uses stdout for
output.

This change fixes test-lz4-basic when run without a terminal
by specifying output filenames.
---
 tests/Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index ebab278..d68c700 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -235,19 +235,19 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	./datagen -g33M   | $(LZ4) --no-frame-crc | $(LZ4) -t
 	./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
 	@echo "hello world" > tmp
-	$(LZ4) --rm -f tmp
+	$(LZ4) --rm -f tmp tmp.lz4
 	test ! -f tmp                      # must fail (--rm)
 	test   -f tmp.lz4
 	$(PRGDIR)/lz4cat tmp.lz4           # must display hello world
 	test   -f tmp.lz4
-	$(PRGDIR)/unlz4 --rm tmp.lz4
+	$(PRGDIR)/unlz4 --rm tmp.lz4 tmp
 	test   -f tmp
 	test ! -f tmp.lz4                  # must fail (--rm)
 	test ! -f tmp.lz4.lz4              # must fail (unlz4)
 	$(PRGDIR)/lz4cat tmp               # pass-through mode
 	test   -f tmp
 	test ! -f tmp.lz4                  # must fail (lz4cat)
-	$(LZ4) tmp                         # creates tmp.lz4
+	$(LZ4) tmp tmp.lz4                 # creates tmp.lz4
 	$(PRGDIR)/lz4cat < tmp.lz4 > tmp3  # checks lz4cat works with stdin (#285)
 	$(DIFF) -q tmp tmp3
 	$(PRGDIR)/lz4cat < tmp > tmp2      # checks lz4cat works with stdin (#285)
-- 
cgit v0.12


From 534f8fa5d6504c00ac7e956a7891ed438af2e212 Mon Sep 17 00:00:00 2001
From: remittor 
Date: Tue, 7 Mar 2017 17:11:48 +0300
Subject: lz4hc: Cleanup function LZ4HC_compress_hashChain

---
 lib/lz4hc.c | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5d4ea3e..2518300 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -398,7 +398,8 @@ _Search3:
 
         if (start2 + ml2 < mflimit)
             ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
-        else ml3 = ml2;
+        else
+            ml3 = ml2;
 
         if (ml3 == ml2) {  /* No better match : 2 sequences to encode */
             /* ip & ref are known; Now for ml */
@@ -474,12 +475,21 @@ _Search3:
     }
 
     /* Encode Last Literals */
-    {   int lastRun = (int)(iend - anchor);
-        if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;  /* Check output limit */
-        if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
-        else *op++ = (BYTE)(lastRun< oend)) return 0;  /* Check output limit */
+        if (lastRunSize >= RUN_MASK) {
+            size_t accumulator = lastRunSize - RUN_MASK;
+            *op++ = (RUN_MASK << ML_BITS);
+            for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
+            *op++ = (BYTE) accumulator;
+        } else {
+            *op++ = (BYTE)(lastRunSize << ML_BITS);
+        }
+        memcpy(op, anchor, lastRunSize);
+        op += lastRunSize;
     }
 
     /* End */
-- 
cgit v0.12


From 45b592b7ecf7afc686a4b9d0ddfcae2f7229f55e Mon Sep 17 00:00:00 2001
From: remittor 
Date: Wed, 8 Mar 2017 00:30:54 +0300
Subject: lz4hc: Cleanup function LZ4HC_encodeSequence

---
 lib/lz4hc.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2518300..d3cc831 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -261,10 +261,10 @@ FORCE_INLINE int LZ4HC_encodeSequence (
     const BYTE** anchor,
     int matchLength,
     const BYTE* const match,
-    limitedOutput_directive limitedOutputBuffer,
+    limitedOutput_directive limit,
     BYTE* oend)
 {
-    int length;
+    size_t length;
     BYTE* token;
 
 #if LZ4HC_DEBUG
@@ -272,11 +272,18 @@ FORCE_INLINE int LZ4HC_encodeSequence (
 #endif
 
     /* Encode Literal length */
-    length = (int)(*ip - *anchor);
+    length = (size_t)(*ip - *anchor);
     token = (*op)++;
-    if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1;   /* Check output limit */
-    if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255;  *(*op)++ = (BYTE)len; }
-    else *token = (BYTE)(length<> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1;   /* Check output limit */
+    if (length >= RUN_MASK) { 
+        size_t len;
+        *token = (RUN_MASK << ML_BITS);
+        len = length - RUN_MASK;
+        for(; len >= 255 ; len -= 255) *(*op)++ = 255;
+        *(*op)++ = (BYTE)len;
+    } else {
+        *token = (BYTE)(length << ML_BITS);
+    }
 
     /* Copy Literals */
     LZ4_wildCopy(*op, *anchor, (*op) + length);
@@ -286,13 +293,13 @@ FORCE_INLINE int LZ4HC_encodeSequence (
     LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
 
     /* Encode MatchLength */
-    length = (int)(matchLength-MINMATCH);
-    if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1;   /* Check output limit */
-    if (length>=(int)ML_MASK) {
+    length = (size_t)(matchLength - MINMATCH);
+    if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1;   /* Check output limit */
+    if (length >= ML_MASK) {
         *token += ML_MASK;
         length -= ML_MASK;
-        for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; }
-        if (length > 254) { length-=255; *(*op)++ = 255; }
+        for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; }
+        if (length >= 255) { length -= 255; *(*op)++ = 255; }
         *(*op)++ = (BYTE)length;
     } else {
         *token += (BYTE)(length);
-- 
cgit v0.12


From f007153e3f0c401d51d6ba5f6ecf27b223591ef2 Mon Sep 17 00:00:00 2001
From: remittor 
Date: Wed, 8 Mar 2017 11:11:15 +0300
Subject: lz4hc: Add LZ4_compressHC_destSize and
 LZ4_compress_HC_continue_destSize

---
 lib/lz4hc.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++---------
 lib/lz4hc.h |  2 ++
 2 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d3cc831..abea72e 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -248,7 +248,11 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 }
 
 
-typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum {
+    noLimit = 0,
+    limitedOutput = 1,
+    limitedDestSize = 2,
+} limitedOutput_directive;
 
 #define LZ4HC_DEBUG 0
 #if LZ4HC_DEBUG
@@ -318,18 +322,21 @@ static int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
     char* const dest,
-    int const inputSize,
+    int* srcSizePtr,
     int const maxOutputSize,
     unsigned maxNbAttempts,
     limitedOutput_directive limit
     )
 {
+    const int inputSize = *srcSizePtr;
+
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimit = iend - MFLIMIT;
     const BYTE* const matchlimit = (iend - LASTLITERALS);
 
+    BYTE* optr = (BYTE*) dest;
     BYTE* op = (BYTE*) dest;
     BYTE* const oend = op + maxOutputSize;
 
@@ -343,7 +350,12 @@ static int LZ4HC_compress_hashChain (
     const BYTE* ref0;
 
     /* init */
+    *srcSizePtr = 0;
+    if (limit && maxOutputSize < 1) return 0;                            /* Impossible to store anything */
+    if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;              /* Unsupported input size, too large (or negative) */
+
     ctx->end += inputSize;
+    if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     ip++;
 
@@ -363,7 +375,8 @@ _Search2:
         else ml2 = ml;
 
         if (ml2 == ml) { /* No better match */
-            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+            optr = op;
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
             continue;
         }
 
@@ -412,9 +425,11 @@ _Search3:
             /* ip & ref are known; Now for ml */
             if (start2 < ip+ml)  ml = (int)(start2 - ip);
             /* Now, encode 2 sequences */
-            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+            optr = op;
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
             ip = start2;
-            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
+            optr = op;
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow;
             continue;
         }
 
@@ -432,7 +447,8 @@ _Search3:
                     }
                 }
 
-                if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+                optr = op;
+                if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
                 ip  = start3;
                 ref = ref3;
                 ml  = ml3;
@@ -468,7 +484,8 @@ _Search3:
                 ml = (int)(start2 - ip);
             }
         }
-        if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+        optr = op;
+        if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow;
 
         ip = start2;
         ref = ref2;
@@ -481,12 +498,21 @@ _Search3:
         goto _Search3;
     }
 
+_last_literals:
     /* Encode Last Literals */
     {   size_t lastRunSize, litLength, totalSize;
         lastRunSize = (size_t)(iend - anchor);  /* literals */
         litLength = (lastRunSize + 255 - RUN_MASK) / 255;
         totalSize = 1 + litLength + lastRunSize;
-        if ((limit == limitedOutput) && (op + totalSize > oend)) return 0;  /* Check output limit */
+        if (limit && (op + totalSize > oend)) {
+            if (limit == limitedOutput) return 0;  /* Check output limit */
+            /* adapt lastRunSize to fill 'dst' */
+            lastRunSize  = (size_t)(oend - op) - 1;
+            litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+            lastRunSize -= litLength;
+        }
+        ip = anchor + lastRunSize;
+
         if (lastRunSize >= RUN_MASK) {
             size_t accumulator = lastRunSize - RUN_MASK;
             *op++ = (RUN_MASK << ML_BITS);
@@ -500,7 +526,15 @@ _Search3:
     }
 
     /* End */
+    *srcSizePtr = (int) (((const char*)ip) - source);
     return (int) (((char*)op)-dest);
+
+_dest_overflow:
+    if (limit == limitedDestSize) {
+        op = optr;  /* restore correct out pointer */
+        goto _last_literals;
+    }
+    return 0;
 }
 
 static int LZ4HC_getSearchNum(int compressionLevel)
@@ -522,16 +556,17 @@ static int LZ4HC_compress_generic (
     limitedOutput_directive limit
     )
 {
+    int srcSize = inputSize;
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > 9) {
         switch (compressionLevel) {
-            case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
+            case 10: return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit);
             case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
             default:
             case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
         }
     }
-    return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
+    return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (compressionLevel-1), limit);
 }
 
 
@@ -657,6 +692,17 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc
         return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
 }
 
+int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize)
+{
+    LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    unsigned maxNbAttempts = 1 << (ctxPtr->compressionLevel - 1);
+
+    /* destSize: always auto-init */
+    LZ4HC_init(ctxPtr, (const BYTE*) source);
+
+    return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
+}
+
 
 /* dictionary saving */
 
@@ -735,3 +781,11 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
     return (char*)(hc4->inputBuffer + dictSize);
 }
+
+int LZ4_compressHC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel)
+{
+    LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    unsigned maxNbAttempts = 1 << (compressionLevel - 1);
+    LZ4HC_init(ctx, (const BYTE*) source);
+    return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
+}
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 1036fd0..13112f9 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -99,6 +99,7 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
 LZ4LIB_API int  LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
 
 LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
+LZ4LIB_API int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int maxDstSize);
 
 LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
 
@@ -219,6 +220,7 @@ LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_con
 LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
 LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
 LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue_destSize() instead") int LZ4_compressHC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel);
 
 
 #if defined (__cplusplus)
-- 
cgit v0.12


From 66b26a389f210548c49a69983ff4ffd2250bf0d0 Mon Sep 17 00:00:00 2001
From: remittor 
Date: Wed, 8 Mar 2017 11:13:28 +0300
Subject: tests: fuzzer: Add test for LZ4_compressHC_destSize

---
 tests/fuzzer.c | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index b129c96..6da0ea5 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -32,6 +32,7 @@
 #  pragma warning(disable : 4310)    /* disable: C4310: constant char value > 127 */
 #endif
 
+#define LZ4_DISABLE_DEPRECATE_WARNINGS
 
 /*-************************************
 *  Dependencies
@@ -348,6 +349,41 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 DISPLAYLEVEL(5, " \n");
         }
 
+        /* Test compression HC destSize */
+        FUZ_DISPLAYTEST;
+        {   int srcSize = blockSize;
+            int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
+            char endCheck = FUZ_rand(&randState) & 255;
+            void * ctx = LZ4_createHC(block);
+            FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
+            compressedBuffer[targetSize] = endCheck;
+            ret = LZ4_compressHC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
+            LZ4_freeHC(ctx);
+            FUZ_CHECKTEST(ret > targetSize, "LZ4_compressHC_destSize() result larger than dst buffer !");
+            FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compressHC_destSize() overwrite dst buffer !");
+            FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compressHC_destSize() fed more than src buffer !");
+            DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
+            if (targetSize>0) {
+                /* check correctness */
+                U32 const crcBase = XXH32(block, srcSize, 0);
+                char const canary = FUZ_rand(&randState) & 255;
+                FUZ_CHECKTEST((ret==0), "LZ4_compressHC_destSize() compression failed");
+                FUZ_DISPLAYTEST;
+                compressedSize = ret;
+                decodedBuffer[srcSize] = canary;
+                ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
+                FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
+                FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+                FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
+                { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
+                  FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
+
+                DISPLAYLEVEL(5, " OK \n");
+            }
+            else
+                DISPLAYLEVEL(5, " \n");
+        }
+
         /* Test compression HC */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
@@ -688,7 +724,7 @@ static void FUZ_unitTests(int compressionLevel)
         crcOrig = XXH64(testInput, testCompressedSize, 0);
         LZ4_resetStream(&streamingState);
         result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
-        FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
+        FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!");
 
         result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
         FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
@@ -718,7 +754,7 @@ static void FUZ_unitTests(int compressionLevel)
 
                 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
                 result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
-                FUZ_CHECKTEST(result==0, "LZ4_compress_limitedOutput_continue() compression failed");
+                FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed");
 
                 result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
                 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
-- 
cgit v0.12


From baa155088b2700526cb97e97b6e2ce4199bf264c Mon Sep 17 00:00:00 2001
From: remittor 
Date: Wed, 8 Mar 2017 18:49:55 +0300
Subject: lz4hc: Fix LZ4HC_compress_hashChain for full support destSize variant

---
 lib/lz4hc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index abea72e..618ed8a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -338,7 +338,7 @@ static int LZ4HC_compress_hashChain (
 
     BYTE* optr = (BYTE*) dest;
     BYTE* op = (BYTE*) dest;
-    BYTE* const oend = op + maxOutputSize;
+    BYTE* oend = op + maxOutputSize;
 
     int   ml, ml2, ml3, ml0;
     const BYTE* ref = NULL;
@@ -355,6 +355,7 @@ static int LZ4HC_compress_hashChain (
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;              /* Unsupported input size, too large (or negative) */
 
     ctx->end += inputSize;
+    if (limit == limitedDestSize) oend -= LASTLITERALS;                  /* Hack for support limitations LZ4 decompressor */
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     ip++;
@@ -372,7 +373,8 @@ static int LZ4HC_compress_hashChain (
 _Search2:
         if (ip+ml < mflimit)
             ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts);
-        else ml2 = ml;
+        else
+            ml2 = ml;
 
         if (ml2 == ml) { /* No better match */
             optr = op;
@@ -504,6 +506,7 @@ _last_literals:
         lastRunSize = (size_t)(iend - anchor);  /* literals */
         litLength = (lastRunSize + 255 - RUN_MASK) / 255;
         totalSize = 1 + litLength + lastRunSize;
+        if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
         if (limit && (op + totalSize > oend)) {
             if (limit == limitedOutput) return 0;  /* Check output limit */
             /* adapt lastRunSize to fill 'dst' */
-- 
cgit v0.12


From 36842ebb1978a3e500572ead564e60e513c4ad0a Mon Sep 17 00:00:00 2001
From: remittor 
Date: Thu, 9 Mar 2017 12:19:24 +0300
Subject: lz4hc: Fix LZ4HC_compress_hashChain for backward compatibility

---
 lib/lz4hc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 618ed8a..290e575 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -351,7 +351,7 @@ static int LZ4HC_compress_hashChain (
 
     /* init */
     *srcSizePtr = 0;
-    if (limit && maxOutputSize < 1) return 0;                            /* Impossible to store anything */
+    if (limit == limitedDestSize && maxOutputSize < 1) return 0;         /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;              /* Unsupported input size, too large (or negative) */
 
     ctx->end += inputSize;
@@ -509,7 +509,7 @@ _last_literals:
         if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
         if (limit && (op + totalSize > oend)) {
             if (limit == limitedOutput) return 0;  /* Check output limit */
-            /* adapt lastRunSize to fill 'dst' */
+            /* adapt lastRunSize to fill 'dest' */
             lastRunSize  = (size_t)(oend - op) - 1;
             litLength = (lastRunSize + 255 - RUN_MASK) / 255;
             lastRunSize -= litLength;
-- 
cgit v0.12


From cb1671955a056302c804ff09d6edf0e095566274 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 10 Mar 2017 20:49:25 +0100
Subject: bench.c: respect LZ4_MAX_INPUT_SIZE limit

---
 programs/bench.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 71d3896..77a9e3f 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -455,8 +455,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
     benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
     if (benchedSize==0) EXM_THROW(12, "not enough memory");
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
-    if (benchedSize < totalSizeToLoad)
-        DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+    if (benchedSize > LZ4_MAX_INPUT_SIZE) {
+        benchedSize = LZ4_MAX_INPUT_SIZE; 
+        DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20));
+    } else { 
+        if (benchedSize < totalSizeToLoad)
+            DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+    }
     srcBuffer = malloc(benchedSize + !benchedSize);   /* avoid alloc of zero */
     if (!srcBuffer) EXM_THROW(12, "not enough memory");
 
-- 
cgit v0.12


From 7aeecbff7178d72da1786ed30817fe8071c4d9ea Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Fri, 10 Mar 2017 23:35:30 +0000
Subject: Explicitly set visibility of public API functions when gcc is used

Windows builds already limit exporting of functions to those marked
with LZ4LIB_API tag.  The same behaviour could be achieved on other
platforms when a relatively fresh gcc is used.

This change assigns public visibility to all symbols marked with
LZ4LIB_API tag.  When the library is built in -fvisibility=hidden
mode, only these marked symbols will be exported.
---
 lib/lz4.h      | 4 ++++
 lib/lz4frame.h | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/lib/lz4.h b/lib/lz4.h
index bebe761..b0537fc 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -72,11 +72,15 @@ extern "C" {
 /*
 *  LZ4_DLL_EXPORT :
 *  Enable exporting of functions when building a Windows DLL
+*  LZ4LIB_API :
+*  Control library symbols visibility.
 */
 #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
 #  define LZ4LIB_API __declspec(dllexport)
 #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
 #  define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+#  define LZ4LIB_API __attribute__ ((__visibility__ ("default")))
 #else
 #  define LZ4LIB_API
 #endif
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9d9fe89..f33eee1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -63,11 +63,15 @@ extern "C" {
 /*
 *  LZ4_DLL_EXPORT :
 *  Enable exporting of functions when building a Windows DLL
+*  LZ4FLIB_API :
+*  Control library symbols visibility.
 */
 #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
 #  define LZ4FLIB_API __declspec(dllexport)
 #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
 #  define LZ4FLIB_API __declspec(dllimport)
+#elif defined(__GNUC__) && (__GNUC__ >= 4)
+#  define LZ4FLIB_API __attribute__ ((__visibility__ ("default")))
 #else
 #  define LZ4FLIB_API
 #endif
-- 
cgit v0.12


From 25b243588585b06a2d947372284cbfe00da930e9 Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Fri, 10 Mar 2017 23:35:30 +0000
Subject: Export deprecated symbols

Deprecated symbols are still a part of ABI and have to be exported,
so mark them with LZ4LIB_API attribute.
---
 lib/lz4.h | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index b0537fc..fa8312e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -435,12 +435,12 @@ union LZ4_streamDecode_u {
 #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
 
 /* Obsolete compression functions */
-LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress               (const char* source, char* dest, int sourceSize);
-LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress               (const char* source, char* dest, int sourceSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Obsolete decompression functions */
 /* These function names are completely deprecated and must no longer be used.
@@ -453,14 +453,14 @@ LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limi
 /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
 
 /* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
 
 /* Obsolete streaming decoding functions */
-LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
 
 
 #if defined (__cplusplus)
-- 
cgit v0.12


From 883ebdcee0aae3e8339dd3658c794ca49dfb18b1 Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Fri, 10 Mar 2017 23:35:30 +0000
Subject: Export only those symbols that are part of public API

Specify -fvisibility=hidden parameter when linking the shared library
using -fPIC, assuming that gcc >= 4 is used.

This change results to unexporting of the following 42 functions:

LZ4F_getErrorCode
LZ4_XXH32
LZ4_XXH32_canonicalFromHash
LZ4_XXH32_copyState
LZ4_XXH32_createState
LZ4_XXH32_digest
LZ4_XXH32_freeState
LZ4_XXH32_hashFromCanonical
LZ4_XXH32_reset
LZ4_XXH32_update
LZ4_XXH64
LZ4_XXH64_canonicalFromHash
LZ4_XXH64_copyState
LZ4_XXH64_createState
LZ4_XXH64_digest
LZ4_XXH64_freeState
LZ4_XXH64_hashFromCanonical
LZ4_XXH64_reset
LZ4_XXH64_update
LZ4_XXH_versionNumber
LZ4_compressHC
LZ4_compressHC2
LZ4_compressHC2_continue
LZ4_compressHC2_limitedOutput
LZ4_compressHC2_limitedOutput_continue
LZ4_compressHC2_limitedOutput_withStateHC
LZ4_compressHC2_withStateHC
LZ4_compressHC_continue
LZ4_compressHC_limitedOutput
LZ4_compressHC_limitedOutput_continue
LZ4_compressHC_limitedOutput_withStateHC
LZ4_compressHC_withStateHC
LZ4_compress_fast_force
LZ4_compress_forceExtDict
LZ4_createHC
LZ4_decompress_safe_forceExtDict
LZ4_freeHC
LZ4_resetStreamStateHC
LZ4_sizeofStreamStateHC
LZ4_slideInputBufferHC
LZ4_uncompress
LZ4_uncompress_unknownOutputSize
---
 lib/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index c4bc7d2..9a794b8 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -95,7 +95,7 @@ ifneq (,$(filter Windows%,$(OS)))
 	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
 	dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib
 else
-	@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@
+	@$(CC) $(FLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
 	@ln -sf $@ liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf $@ liblz4.$(SHARED_EXT)
-- 
cgit v0.12


From e1b2b370e195f1bb6930a7b152a237adfb018e58 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 12 Mar 2017 03:18:34 -0700
Subject: added Oleg @remittor contribution

---
 NEWS | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS b/NEWS
index 8075947..d348dd1 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,8 @@
 v1.7.6
+API : added LZ4_compressHC_destSize(), by Oleg (@remittor)
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
 
+
 v1.7.5
 lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski
 lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z)
-- 
cgit v0.12


From e945a27f9c7081349fdcd292ea870bc240f667fd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 14 Mar 2017 21:21:54 -0700
Subject: fix #333 : expose obsolete decoding functions

with deprecation warning.
notified by Chen Yufei
---
 lib/lz4.h | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index fa8312e..efc92af 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -443,14 +443,8 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_co
 LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Obsolete decompression functions */
-/* These function names are completely deprecated and must no longer be used.
-   They are only provided in lz4.c for compatibility with older programs.
-    - LZ4_uncompress is the same as LZ4_decompress_fast
-    - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
-   These function prototypes are now disabled; uncomment them only if you really need them.
-   It is highly recommended to stop using these prototypes and migrate to maintained ones */
-/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
-/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
 
 /* Obsolete streaming functions; use new streaming interface whenever possible */
 LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-- 
cgit v0.12


From a33bf89c62b1573a604144adaa797d94d97cb990 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Mar 2017 17:20:22 -0700
Subject: fix #332 : do not modify /dev/null permissions

---
 NEWS             |  4 ++++
 programs/lz4io.c | 68 ++++++++++++++++++++++++++------------------------------
 programs/util.h  | 10 ++++-----
 3 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/NEWS b/NEWS
index d348dd1..61e13d3 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,10 @@
 v1.7.6
+cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
 API : added LZ4_compressHC_destSize(), by Oleg (@remittor)
+API : fix : expose obsolete decoding functions, reported by Chen Yufei
+build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
+doc : lz4 api manual, by Przemyslaw Skibinski
 
 
 v1.7.5
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 5264b14..5bf5474 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -222,7 +222,7 @@ static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ
 
 
 /** LZ4IO_openSrcFile() :
- * condition : `dstFileName` must be non-NULL.
+ * condition : `srcFileName` must be non-NULL.
  * @result : FILE* to `dstFileName`, or NULL if it fails */
 static FILE* LZ4IO_openSrcFile(const char* srcFileName)
 {
@@ -291,7 +291,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName)
 /* unoptimized version; solves endianess & alignment issues */
 static void LZ4IO_writeLE32 (void* p, unsigned value32)
 {
-    unsigned char* dstPtr = (unsigned char*)p;
+    unsigned char* const dstPtr = (unsigned char*)p;
     dstPtr[0] = (unsigned char)value32;
     dstPtr[1] = (unsigned char)(value32 >> 8);
     dstPtr[2] = (unsigned char)(value32 >> 16);
@@ -317,10 +317,10 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
     const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
     FILE* finput;
     FILE* foutput;
-    clock_t end;
+    clock_t clockEnd;
 
     /* Init */
-    clock_t const start = clock();
+    clock_t const clockStart = clock();
     if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC;
 
     finput = LZ4IO_openSrcFile(input_filename);
@@ -336,7 +336,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
     /* Write Archive Header */
     LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
     { size_t const sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
-      if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); }
+      if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); }
 
     /* Main Loop */
     while (1) {
@@ -360,13 +360,13 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
     if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
 
     /* Status */
-    end = clock();
-    if (end==start) end+=1;  /* avoid division by zero (speed) */
+    clockEnd = clock();
+    if (clockEnd==clockStart) clockEnd+=1;  /* avoid division by zero (speed) */
     filesize += !filesize;   /* avoid division by zero (ratio) */
     DISPLAYLEVEL(2, "\r%79s\r", "");   /* blank line */
     DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
         filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
-    {   double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
+    {   double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
         DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
     }
 
@@ -603,11 +603,11 @@ static unsigned LZ4IO_readLE32 (const void* s)
     return value32;
 }
 
-#define sizeT sizeof(size_t)
-#define maskT (sizeT - 1)
 
 static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
 {
+    const size_t sizeT = sizeof(size_t);
+    const size_t maskT = sizeT -1 ;
     const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
     const size_t* ptrT = bufferT;
     size_t bufferSizeT = bufferSize / sizeT;
@@ -682,22 +682,19 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 }
 
 
-static unsigned g_magicRead = 0;
+static unsigned g_magicRead = 0;   /* out-parameter of LZ4IO_decodeLegacyStream() */
 static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
 {
-    unsigned long long filesize = 0;
-    char* in_buff;
-    char* out_buff;
+    unsigned long long streamSize = 0;
     unsigned storedSkips = 0;
 
     /* Allocate Memory */
-    in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
-    out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
+    char* const in_buff  = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+    char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
     if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
 
     /* Main Loop */
     while (1) {
-        int decodeSize;
         unsigned int blockSize;
 
         /* Block Size */
@@ -716,13 +713,12 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
           if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
 
         /* Decode Block */
-        decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
-        if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
-        filesize += decodeSize;
-
-        /* Write Block */
-        storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips);
-    }
+        {   int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
+            if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
+            streamSize += decodeSize;
+            /* Write Block */
+            storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */
+    }   }
     if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
 
     LZ4IO_fwriteSparseEnd(foutput, storedSkips);
@@ -731,7 +727,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
     free(in_buff);
     free(out_buff);
 
-    return filesize;
+    return streamSize;
 }
 
 
@@ -925,10 +921,9 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
 {
     FILE* const foutput = ress.dstFile;
     unsigned long long filesize = 0, decodedSize=0;
-    FILE* finput;
 
     /* Init */
-    finput = LZ4IO_openSrcFile(input_filename);
+    FILE* const finput = LZ4IO_openSrcFile(input_filename);
     if (finput==NULL) return 1;
 
     /* Loop over multiple streams */
@@ -954,10 +949,7 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
 
 static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
 {
-    FILE* foutput;
-
-    /* Init */
-    foutput = LZ4IO_openDstFile(output_filename);
+    FILE* const foutput = LZ4IO_openDstFile(output_filename);   /* success or die */
     if (foutput==NULL) return 1;   /* failure */
 
     ress.dstFile = foutput;
@@ -967,8 +959,11 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con
 
     /* Copy owner, file permissions and modification time */
     {   stat_t statbuf;
-        if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf))
-            UTIL_setFileStat(output_filename, &statbuf);
+        if (strcmp (input_filename, stdinmark)
+            && strcmp (output_filename, stdoutmark)
+            && strcmp (output_filename, nulmark)
+            && UTIL_getFileStat(input_filename, &statbuf))
+            UTIL_setFileStat(output_filename, &statbuf);  /* should return value be read ? or is silent fail good enough ? */
     }
 
     return 0;
@@ -982,10 +977,9 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
 
     int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename);
 
-    {   clock_t const end = clock();
-        double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
-        DISPLAYLEVEL(4, "Done in %.2f sec  \n", seconds);
-    }
+    clock_t const end = clock();
+    double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
+    DISPLAYLEVEL(4, "Done in %.2f sec  \n", seconds);
 
     LZ4IO_freeDResources(ress);
     return missingFiles;
diff --git a/programs/util.h b/programs/util.h
index 044085f..5a69c55 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -187,8 +187,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
 *  File functions
 ******************************************/
 #if defined(_MSC_VER)
-	#define chmod _chmod
-	typedef struct __stat64 stat_t;
+    #define chmod _chmod
+    typedef struct __stat64 stat_t;
 #else
     typedef struct stat stat_t;
 #endif
@@ -199,9 +199,9 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
     int res = 0;
     struct utimbuf timebuf;
 
-	timebuf.actime = time(NULL);
-	timebuf.modtime = statbuf->st_mtime;
-	res += utime(filename, &timebuf);  /* set access and modification times */
+    timebuf.actime = time(NULL);
+    timebuf.modtime = statbuf->st_mtime;
+    res += utime(filename, &timebuf);  /* set access and modification times */
 
 #if !defined(_WIN32)
     res += chown(filename, statbuf->st_uid, statbuf->st_gid);  /* Copy ownership */
-- 
cgit v0.12


From 9144520dc986db9fffcdc9e2afecdb9db226a756 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 15 Mar 2017 18:59:26 -0700
Subject: fixed Visual compilation error

static const must be a "constant", like a macro
even if it can be determined at compile time ...
---
 programs/lz4io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 5bf5474..d512782 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -612,7 +612,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer
     const size_t* ptrT = bufferT;
     size_t bufferSizeT = bufferSize / sizeT;
     const size_t* const bufferTEnd = bufferT + bufferSizeT;
-    static const size_t segmentSizeT = (32 KB) / sizeT;
+    const size_t segmentSizeT = (32 KB) / sizeT;
 
     if (!g_sparseFileSupport) {  /* normal write */
         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
-- 
cgit v0.12


From 430b7d32b3f804ece3be95f0b36fb584947d6c73 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Mar 2017 02:16:24 -0700
Subject: created LZ4_HC_STATIC_LINKING_ONLY section

where are exposed new prototypes *_destSize()
---
 lib/lz4hc.c    | 99 +++++++++++++++++++++++++---------------------------------
 lib/lz4hc.h    | 57 +++++++++++++++++++++++++++------
 tests/fuzzer.c |  7 +++--
 3 files changed, 94 insertions(+), 69 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 290e575..b1e77ea 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -38,51 +38,37 @@
 *  Tuning Parameter
 ***************************************/
 
-/*!
- * HEAPMODE :
- * Select how default compression function will allocate workplace memory,
- * in stack (0:fastest), or in heap (1:requires malloc()).
- * Since workplace is rather large, heap mode is recommended.
+/*! HEAPMODE :
+ *  Select how default compression function will allocate workplace memory,
+ *  in stack (0:fastest), or in heap (1:requires malloc()).
+ *  Since workplace is rather large, heap mode is recommended.
  */
 #ifndef LZ4HC_HEAPMODE
 #  define LZ4HC_HEAPMODE 1
 #endif
 
 
-/* *************************************
-*  Dependency
-***************************************/
+/*===    Dependency    ===*/
 #include "lz4hc.h"
 
 
-/* *************************************
-*  Local Compiler Options
-***************************************/
+/*===   Common LZ4 definitions   ===*/
 #if defined(__GNUC__)
 #  pragma GCC diagnostic ignored "-Wunused-function"
 #endif
-
 #if defined (__clang__)
 #  pragma clang diagnostic ignored "-Wunused-function"
 #endif
 
-
-/* *************************************
-*  Common LZ4 definition
-***************************************/
 #define LZ4_COMMONDEFS_ONLY
 #include "lz4.c"
 
 
-/* *************************************
-*  Local Constants
-***************************************/
+/*===   Constants   ===*/
 #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
 
 
-/**************************************
-*  Local Macros
-**************************************/
+/*===   Macros   ===*/
 #define HASH_FUNCTION(i)       (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
 #define DELTANEXTMAXD(p)       chainTable[(p) & LZ4HC_MAXD_MASK]    /* flexible, LZ4HC_MAXD dependent */
 #define DELTANEXTU16(p)        chainTable[(U16)(p)]   /* faster */
@@ -141,8 +127,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4,   /* In
     const U32 dictLimit = hc4->dictLimit;
     const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
     U32 matchIndex;
-    int nbAttempts=maxNbAttempts;
-    size_t ml=0;
+    int nbAttempts = maxNbAttempts;
+    size_t ml = 0;
 
     /* HC4 match finder */
     LZ4HC_Insert(hc4, ip);
@@ -223,9 +209,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                         longest = (int)mlt;
                         *matchpos = matchPtr+back;
                         *startpos = ip+back;
-                    }
-                }
-            }
+            }   }   }
         } else {
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
@@ -279,10 +263,9 @@ FORCE_INLINE int LZ4HC_encodeSequence (
     length = (size_t)(*ip - *anchor);
     token = (*op)++;
     if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1;   /* Check output limit */
-    if (length >= RUN_MASK) { 
-        size_t len;
+    if (length >= RUN_MASK) {
+        size_t len = length - RUN_MASK;
         *token = (RUN_MASK << ML_BITS);
-        len = length - RUN_MASK;
         for(; len >= 255 ; len -= 255) *(*op)++ = 255;
         *(*op)++ = (BYTE)len;
     } else {
@@ -316,8 +299,10 @@ FORCE_INLINE int LZ4HC_encodeSequence (
     return 0;
 }
 
+/* btopt */
 #include "lz4opt.h"
 
+
 static int LZ4HC_compress_hashChain (
     LZ4HC_CCtx_internal* const ctx,
     const char* const source,
@@ -399,11 +384,9 @@ _Search2:
         }
 
 _Search3:
-        /*
-        * Currently we have :
-        * ml2 > ml1, and
-        * ip1+3 <= ip2 (usually < ip1+ml1)
-        */
+        /* At this stage, we have :
+        *  ml2 > ml1, and
+        *  ip1+3 <= ip2 (usually < ip1+ml1) */
         if ((start2 - ip) < OPTIMAL_ML) {
             int correction;
             int new_ml = ml;
@@ -502,10 +485,9 @@ _Search3:
 
 _last_literals:
     /* Encode Last Literals */
-    {   size_t lastRunSize, litLength, totalSize;
-        lastRunSize = (size_t)(iend - anchor);  /* literals */
-        litLength = (lastRunSize + 255 - RUN_MASK) / 255;
-        totalSize = 1 + litLength + lastRunSize;
+    {   size_t lastRunSize = (size_t)(iend - anchor);  /* literals */
+        size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+        size_t const totalSize = 1 + litLength + lastRunSize;
         if (limit == limitedDestSize) oend += LASTLITERALS;  /* restore correct value */
         if (limit && (op + totalSize > oend)) {
             if (limit == limitedOutput) return 0;  /* Check output limit */
@@ -544,8 +526,8 @@ static int LZ4HC_getSearchNum(int compressionLevel)
 {
     switch (compressionLevel) {
         default: return 0; /* unused */
-        case 11: return 128; 
-        case 12: return 1<<10; 
+        case 11: return 128;
+        case 12: return 1<<10;
     }
 }
 
@@ -563,10 +545,15 @@ static int LZ4HC_compress_generic (
     if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
     if (compressionLevel > 9) {
         switch (compressionLevel) {
-            case 10: return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit);
-            case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
+            case 10:
+                return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit);
+            case 11:
+                ctx->searchNum = LZ4HC_getSearchNum(compressionLevel);
+                return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
             default:
-            case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
+            case 12:
+                ctx->searchNum = LZ4HC_getSearchNum(compressionLevel);
+                return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
         }
     }
     return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (compressionLevel-1), limit);
@@ -577,7 +564,7 @@ int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
 
 int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
 {
-    LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
+    LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
     LZ4HC_init (ctx, (const BYTE*)src);
     if (maxDstSize < LZ4_compressBound(srcSize))
@@ -622,7 +609,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
-    LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
     if (dictSize > 64 KB) {
         dictionary += dictSize - 64 KB;
         dictSize = 64 KB;
@@ -659,7 +646,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
                                             const char* source, char* dest,
                                             int inputSize, int maxOutputSize, limitedOutput_directive limit)
 {
-    LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
     /* auto-init if forgotten */
     if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source);
 
@@ -697,7 +684,7 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc
 
 int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize)
 {
-    LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+    LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
     unsigned maxNbAttempts = 1 << (ctxPtr->compressionLevel - 1);
 
     /* destSize: always auto-init */
@@ -706,6 +693,14 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
     return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
 }
 
+int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel)
+{
+    LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    unsigned maxNbAttempts = 1 << (compressionLevel - 1);
+    LZ4HC_init(ctx, (const BYTE*) source);
+    return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
+}
+
 
 /* dictionary saving */
 
@@ -784,11 +779,3 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
     int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
     return (char*)(hc4->inputBuffer + dictSize);
 }
-
-int LZ4_compressHC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel)
-{
-    LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    unsigned maxNbAttempts = 1 << (compressionLevel - 1);
-    LZ4HC_init(ctx, (const BYTE*) source);
-    return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
-}
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 13112f9..04b1d64 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -99,7 +99,6 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL
 LZ4LIB_API int  LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
 
 LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
-LZ4LIB_API int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int maxDstSize);
 
 LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
 
@@ -114,25 +113,23 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 
   Then, use LZ4_compress_HC_continue() to compress each successive block.
   Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
-  'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success.
+  'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success.
+  Because in case of failure, the API does not guarantee context recovery, and context will have to be reset.
+  If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead.
 
-  If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
-  you must save it to a safer memory space, using LZ4_saveDictHC().
+  If, for any reason, previous data block can't be preserved unmodified in memory for next compression block,
+  you can save it to a more stable memory space, using LZ4_saveDictHC().
   Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
 */
 
 
-/*-******************************************
- * !!!!!   STATIC LINKING ONLY   !!!!!
- *******************************************/
-
  /*-*************************************
  * PRIVATE DEFINITIONS :
  * Do not use these definitions.
  * They are exposed to allow static allocation of `LZ4_streamHC_t`.
  * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
  **************************************/
-#define LZ4HC_DICTIONARY_LOGSIZE 17
+#define LZ4HC_DICTIONARY_LOGSIZE 17   /* because of btopt, hc would only need 16 */
 #define LZ4HC_MAXD (1<      /* fgets, sscanf */
 #include      /* strcmp */
 #include        /* clock_t, clock, CLOCKS_PER_SEC */
+#define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"
@@ -353,11 +354,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_DISPLAYTEST;
         {   int srcSize = blockSize;
             int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
-            char endCheck = FUZ_rand(&randState) & 255;
-            void * ctx = LZ4_createHC(block);
+            char const endCheck = FUZ_rand(&randState) & 255;
+            void* ctx = LZ4_createHC(block);
             FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
             compressedBuffer[targetSize] = endCheck;
-            ret = LZ4_compressHC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
+            ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
             LZ4_freeHC(ctx);
             FUZ_CHECKTEST(ret > targetSize, "LZ4_compressHC_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compressHC_destSize() overwrite dst buffer !");
-- 
cgit v0.12


From 6b2a1b3606be01a0b4899a1aaa566f0943064da9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Mar 2017 02:24:46 -0700
Subject: updated lz4 version number

---
 lib/lz4.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index efc92af..25622f3 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -89,7 +89,7 @@ extern "C" {
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    7    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  5    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 
-- 
cgit v0.12


From cd35f0d98c54e3ba0573ce5473482fadd6ec0a59 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Mar 2017 15:10:38 -0700
Subject: LZ4_compress_HC_destSize() uses LZ4HC_compress_generic() code path

Limits compression level to 10,
to remain compatible with Hash Chain.
---
 NEWS             |  2 +-
 lib/lz4.c        |  2 +-
 lib/lz4.h        |  2 +-
 lib/lz4frame.h   |  4 +--
 lib/lz4hc.c      | 92 +++++++++++++++++++++++++++++---------------------------
 lib/lz4hc.h      | 20 ++++++------
 lib/lz4opt.h     |  2 +-
 programs/lz4io.c |  2 +-
 tests/fuzzer.c   | 84 +++++++++++++++++++++++++++++++++------------------
 9 files changed, 119 insertions(+), 91 deletions(-)

diff --git a/NEWS b/NEWS
index 61e13d3..742217f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,6 @@
 v1.7.6
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
-API : added LZ4_compressHC_destSize(), by Oleg (@remittor)
+API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
diff --git a/lib/lz4.c b/lib/lz4.c
index 143c36e..4558117 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1,6 +1,6 @@
 /*
    LZ4 - Fast LZ compression algorithm
-   Copyright (C) 2011-2016, Yann Collet.
+   Copyright (C) 2011-2017, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
diff --git a/lib/lz4.h b/lib/lz4.h
index 25622f3..88d56d0 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -1,7 +1,7 @@
 /*
  *  LZ4 - Fast LZ compression algorithm
  *  Header File
- *  Copyright (C) 2011-2016, Yann Collet.
+ *  Copyright (C) 2011-2017, Yann Collet.
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index f33eee1..7b750b6 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -1,7 +1,7 @@
 /*
    LZ4 auto-framing library
    Header File
-   Copyright (C) 2011-2016, Yann Collet.
+   Copyright (C) 2011-2017, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -168,7 +168,7 @@ typedef struct {
  * All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 count as 0 */
   unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b1e77ea..5f7cfc1 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1,6 +1,6 @@
 /*
     LZ4 HC - High Compression Mode of LZ4
-    Copyright (C) 2011-2016, Yann Collet.
+    Copyright (C) 2011-2017, Yann Collet.
 
     BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
@@ -61,7 +61,7 @@
 #endif
 
 #define LZ4_COMMONDEFS_ONLY
-#include "lz4.c"
+#include "lz4.c"   /* LZ4_count, constants, mem */
 
 
 /*===   Constants   ===*/
@@ -533,47 +533,46 @@ static int LZ4HC_getSearchNum(int compressionLevel)
 
 static int LZ4HC_compress_generic (
     LZ4HC_CCtx_internal* const ctx,
-    const char* const source,
-    char* const dest,
-    int const inputSize,
-    int const maxOutputSize,
-    int compressionLevel,
+    const char* const src,
+    char* const dst,
+    int* const srcSizePtr,
+    int const dstCapacity,
+    int cLevel,
     limitedOutput_directive limit
     )
 {
-    int srcSize = inputSize;
-    if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
-    if (compressionLevel > 9) {
-        switch (compressionLevel) {
+    if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe to reconsider */
+    if (cLevel > 9) {
+        switch (cLevel) {
             case 10:
-                return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (16-1), limit);
+                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (16-1), limit);
             case 11:
-                ctx->searchNum = LZ4HC_getSearchNum(compressionLevel);
-                return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
+                ctx->searchNum = LZ4HC_getSearchNum(cLevel);
+                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
             default:
             case 12:
-                ctx->searchNum = LZ4HC_getSearchNum(compressionLevel);
-                return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
+                ctx->searchNum = LZ4HC_getSearchNum(cLevel);
+                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1);
         }
     }
-    return LZ4HC_compress_hashChain(ctx, source, dest, &srcSize, maxOutputSize, 1 << (compressionLevel-1), limit);
+    return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit);  /* levels 1-9 */
 }
 
 
 int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
 
-int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
     if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
     LZ4HC_init (ctx, (const BYTE*)src);
-    if (maxDstSize < LZ4_compressBound(srcSize))
-        return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
+    if (dstCapacity < LZ4_compressBound(srcSize))
+        return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput);
     else
-        return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
+        return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit);
 }
 
-int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
 {
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t));
@@ -581,13 +580,24 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int
     LZ4_streamHC_t state;
     LZ4_streamHC_t* const statePtr = &state;
 #endif
-    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
+    int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel);
 #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
     free(statePtr);
 #endif
     return cSize;
 }
 
+/* LZ4_compress_HC_destSize() :
+ * currently, only compatible with Hash Chain implementation,
+ * limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/
+int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
+{
+    LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
+    if (cLevel >= LZ4HC_CLEVEL_OPT_MIN) cLevel = LZ4HC_CLEVEL_OPT_MIN-1;   /* this only works with hash chain */
+    LZ4HC_init(ctx, (const BYTE*) source);
+    return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
+}
+
 
 
 /**************************************
@@ -603,7 +613,8 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
+    if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;  /* cap compression level */
+    LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
     LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
 }
 
@@ -671,35 +682,26 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
         }
     }
 
-    return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
+    return LZ4HC_compress_generic (ctxPtr, source, dest, &inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
 }
 
-int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity)
 {
-    if (maxOutputSize < LZ4_compressBound(inputSize))
-        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
+    if (dstCapacity < LZ4_compressBound(srcSize))
+        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, limitedOutput);
     else
-        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
+        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, noLimit);
 }
 
 int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
-    unsigned maxNbAttempts = 1 << (ctxPtr->compressionLevel - 1);
-
-    /* destSize: always auto-init */
-    LZ4HC_init(ctxPtr, (const BYTE*) source);
-
+    int const cLevel = (ctxPtr->compressionLevel < 1) ? 1 : ctxPtr->compressionLevel;  /* guaranteed <= LZ4HC_CLEVEL_MAX */
+    unsigned const maxNbAttempts = 1 << (cLevel - 1);
+    LZ4HC_init(ctxPtr, (const BYTE*) source);   /* LZ4_compress_HC_continue_destSize: always auto-init - only compatible with hash chain - miss continuity check */
     return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
 }
 
-int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int compressionLevel)
-{
-    LZ4HC_CCtx_internal * const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    unsigned maxNbAttempts = 1 << (compressionLevel - 1);
-    LZ4HC_init(ctx, (const BYTE*) source);
-    return LZ4HC_compress_hashChain(ctx, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
-}
 
 
 /* dictionary saving */
@@ -763,14 +765,14 @@ void* LZ4_createHC (char* inputBuffer)
 
 int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
 
-int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
 {
-    return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit);
+    return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, noLimit);
 }
 
-int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel)
 {
-    return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+    return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput);
 }
 
 char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 04b1d64..8535a9e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -1,7 +1,7 @@
 /*
    LZ4 HC - High Compression Mode of LZ4
    Header File
-   Copyright (C) 2011-2016, Yann Collet.
+   Copyright (C) 2011-2017, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -145,14 +145,14 @@ typedef struct
 {
     uint32_t   hashTable[LZ4HC_HASHTABLESIZE];
     uint16_t   chainTable[LZ4HC_MAXD];
-    const uint8_t* end;        /* next block here to continue on current prefix */
-    const uint8_t* base;       /* All index relative to this position */
-    const uint8_t* dictBase;   /* alternate base for extDict */
-    uint8_t* inputBuffer;      /* deprecated */
-    uint32_t   dictLimit;        /* below that point, need extDict */
-    uint32_t   lowLimit;         /* below that point, no more dict */
-    uint32_t   nextToUpdate;     /* index from which to continue dictionary update */
-    uint32_t   searchNum;        /* only for optimal parser */
+    const uint8_t* end;         /* next block here to continue on current prefix */
+    const uint8_t* base;        /* All index relative to this position */
+    const uint8_t* dictBase;    /* alternate base for extDict */
+    uint8_t* inputBuffer;       /* deprecated */
+    uint32_t   dictLimit;       /* below that point, need extDict */
+    uint32_t   lowLimit;        /* below that point, no more dict */
+    uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
+    uint32_t   searchNum;       /* only for optimal parser */
     uint32_t   compressionLevel;
 } LZ4HC_CCtx_internal;
 
@@ -170,7 +170,7 @@ typedef struct
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
     unsigned int   searchNum;        /* only for optimal parser */
-    unsigned int   compressionLevel;
+    int            compressionLevel;
 } LZ4HC_CCtx_internal;
 
 #endif
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index d1913fe..582aa14 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -1,6 +1,6 @@
 /*
     lz4opt.h - Optimal Mode of LZ4
-    Copyright (C) 2015-2016, Przemyslaw Skibinski 
+    Copyright (C) 2015-2017, Przemyslaw Skibinski 
     Note : this file is intended to be included within lz4hc.c
 
     BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
diff --git a/programs/lz4io.c b/programs/lz4io.c
index d512782..e588f7a 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1,6 +1,6 @@
 /*
   LZ4io.c - LZ4 File/Stream Interface
-  Copyright (C) Yann Collet 2011-2016
+  Copyright (C) Yann Collet 2011-2017
 
   GPL v2 License
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c63ecb2..28d2be2 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1,6 +1,6 @@
 /*
     fuzzer.c - Fuzzer test tool for LZ4
-    Copyright (C) Yann Collet 2012-2016
+    Copyright (C) Yann Collet 2012-2017
 
     GPL v2 License
 
@@ -360,15 +360,15 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             compressedBuffer[targetSize] = endCheck;
             ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
             LZ4_freeHC(ctx);
-            FUZ_CHECKTEST(ret > targetSize, "LZ4_compressHC_destSize() result larger than dst buffer !");
-            FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compressHC_destSize() overwrite dst buffer !");
-            FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compressHC_destSize() fed more than src buffer !");
+            FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
+            FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
+            FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
             DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, srcSize, 0);
                 char const canary = FUZ_rand(&randState) & 255;
-                FUZ_CHECKTEST((ret==0), "LZ4_compressHC_destSize() compression failed");
+                FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
                 FUZ_DISPLAYTEST;
                 compressedSize = ret;
                 decodedBuffer[srcSize] = canary;
@@ -388,23 +388,23 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test compression HC */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
         HCcompressedSize = ret;
 
         /* Test compression HC using external state */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
 
         /* Test compression using external state */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
 
         /* Test compression */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
         compressedSize = ret;
 
         /* Decompression tests */
@@ -495,22 +495,22 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Test compression with output size being exactly what's necessary (should work) */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
 
         /* Test compression with output size being exactly what's necessary and external state (should work) */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
-        FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
 
         /* Test HC compression with output size being exactly what's necessary (should work) */
         FUZ_DISPLAYTEST;
         ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
-        FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
+        FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
 
         /* Test compression with missing bytes into output buffer => must fail */
         FUZ_DISPLAYTEST;
@@ -519,8 +519,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
             compressedBuffer[compressedSize-missingBytes] = 0;
             ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
-            FUZ_CHECKTEST(ret, "LZ4_compress_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes);
-            FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes)
+            FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
+            FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes)
         }
 
         /* Test HC compression with missing bytes into output buffer => must fail */
@@ -530,8 +530,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             missingBytes += !missingBytes;   /* avoid special case missingBytes==0 */
             compressedBuffer[HCcompressedSize-missingBytes] = 0;
             ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
-            FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes);
-            FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes)
+            FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
+            FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes)
         }
 
 
@@ -545,14 +545,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             LZ4_resetStream(&LZ4_stream);
             LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1);   /* Just to fill hash tables */
             blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
-            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
         }
 
         /* Decompress with dictionary as prefix */
         FUZ_DISPLAYTEST;
         memcpy(decodedBuffer, dict, dictSize);
         ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
-        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input");
+        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
         crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
         if (crcCheck!=crcOrig) {
             int i=0;
@@ -560,7 +560,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             printf("Wrong Byte at position %i/%i\n", i, blockSize);
 
         }
-        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize);
+        FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
 
         FUZ_DISPLAYTEST;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
@@ -574,18 +574,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
-        FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
+        FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
 
         FUZ_DISPLAYTEST;
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
-        FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
+        FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
         FUZ_DISPLAYTEST;
         LZ4_loadDict(&LZ4dict, dict, dictSize);
         ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
         FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
-        FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
+        FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
 
         /* Decompress with dictionary as external */
         FUZ_DISPLAYTEST;
@@ -633,29 +633,55 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
-        FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
+        FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
 
         FUZ_DISPLAYTEST;
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
-        FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
+        FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
 
         FUZ_DISPLAYTEST;
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
         ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
-        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
-        FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
+        FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
+        FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
 
         FUZ_DISPLAYTEST;
         decodedBuffer[blockSize] = 0;
         ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
         FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
-        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
-            crcCheck = XXH32(decodedBuffer, blockSize, 0);
+        FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        crcCheck = XXH32(decodedBuffer, blockSize, 0);
         if (crcCheck!=crcOrig)
             FUZ_findDiff(block, decodedBuffer);
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
+        /* Compress HC continue destSize */
+            FUZ_DISPLAYTEST;
+        {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
+            int consumedSize = blockSize;
+            FUZ_DISPLAYTEST;
+            LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
+            LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+            blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
+            FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
+            FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
+            DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i \n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace);
+
+            FUZ_DISPLAYTEST;
+            decodedBuffer[consumedSize] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
+            FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
+            FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
+            {   U32 const crcSrc = XXH32(block, consumedSize, 0);
+                U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
+                if (crcSrc!=crcDst)
+                    FUZ_findDiff(block, decodedBuffer);
+                FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
+            }
+        }
+
         /* ***** End of tests *** */
         /* Fill stats */
         bytes += blockSize;
-- 
cgit v0.12


From aae447fffd3c5afd95a94c56ea70c349edad1ebd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 16 Mar 2017 15:41:30 -0700
Subject: LZ4_compress_HC_continue_destSize() works as intended up to level 10

It's incompatible with btopt though,
so cLevel >= 11 feature much reduced performance (degraded mode)
---
 lib/lz4hc.c    | 31 +++++++++++++++----------------
 lib/lz4hc.h    |  2 ++
 tests/fuzzer.c |  2 +-
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5f7cfc1..f3c9946 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -543,6 +543,7 @@ static int LZ4HC_compress_generic (
 {
     if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe to reconsider */
     if (cLevel > 9) {
+        if (limit == limitedDestSize) cLevel = 10;
         switch (cLevel) {
             case 10:
                 return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (16-1), limit);
@@ -589,11 +590,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 
 /* LZ4_compress_HC_destSize() :
  * currently, only compatible with Hash Chain implementation,
- * limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/
+ * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
-    if (cLevel >= LZ4HC_CLEVEL_OPT_MIN) cLevel = LZ4HC_CLEVEL_OPT_MIN-1;   /* this only works with hash chain */
     LZ4HC_init(ctx, (const BYTE*) source);
     return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize);
 }
@@ -654,12 +654,13 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
 }
 
 static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
-                                            const char* source, char* dest,
-                                            int inputSize, int maxOutputSize, limitedOutput_directive limit)
+                                            const char* src, char* dst,
+                                            int* srcSizePtr, int dstCapacity,
+                                            limitedOutput_directive limit)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
     /* auto-init if forgotten */
-    if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source);
+    if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src);
 
     /* Check overflow */
     if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
@@ -669,37 +670,35 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
     }
 
     /* Check if blocks follow each other */
-    if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
+    if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src);
 
     /* Check overlapping input/dictionary space */
-    {   const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+    {   const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr;
         const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
         const BYTE* const dictEnd   = ctxPtr->dictBase + ctxPtr->dictLimit;
-        if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
+        if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) {
             if (sourceEnd > dictEnd) sourceEnd = dictEnd;
             ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
             if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
         }
     }
 
-    return LZ4HC_compress_generic (ctxPtr, source, dest, &inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
+    return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
 }
 
 int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity)
 {
     if (dstCapacity < LZ4_compressBound(srcSize))
-        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, limitedOutput);
+        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput);
     else
-        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, noLimit);
+        return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit);
 }
 
-int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize)
+int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
-    int const cLevel = (ctxPtr->compressionLevel < 1) ? 1 : ctxPtr->compressionLevel;  /* guaranteed <= LZ4HC_CLEVEL_MAX */
-    unsigned const maxNbAttempts = 1 << (cLevel - 1);
-    LZ4HC_init(ctxPtr, (const BYTE*) source);   /* LZ4_compress_HC_continue_destSize: always auto-init - only compatible with hash chain - miss continuity check */
-    return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize);
+    if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src);   /* not compatible with btopt implementation */
+    return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize);
 }
 
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 8535a9e..09a648c 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -258,6 +258,8 @@ LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data,
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
  * `sourceSizePtr` : value will be updated to indicate how much bytes were read from `source`
+ * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
+ *             beyond that level, compression performance will be much reduced due to internal incompatibilities
  */
 LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
                             const char* src, char* dst,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 28d2be2..a36d20b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -664,10 +664,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
             LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
             blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
+            DISPLAYLEVEL(3, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
             FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
             FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
-            DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i \n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace);
 
             FUZ_DISPLAYTEST;
             decodedBuffer[consumedSize] = 0;
-- 
cgit v0.12


From 0d073d4d287b08659eada92c614bec30fa277cd0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 15:11:09 -0700
Subject: added `extern C` for lz4.h static section

should make the file more compatible with C++ compiler, such as Visual or g++
---
 lib/lz4.h      |  9 ++++-----
 lib/lz4frame.c | 43 ++++++++++++++++++++++---------------------
 lib/lz4frame.h | 25 ++++++++++++-------------
 lib/lz4hc.h    |  2 +-
 tests/fuzzer.c |  2 +-
 5 files changed, 40 insertions(+), 41 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 88d56d0..588de22 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -32,13 +32,13 @@
     - LZ4 homepage : http://www.lz4.org
     - LZ4 source repository : https://github.com/lz4/lz4
 */
-#ifndef LZ4_H_2983827168210
-#define LZ4_H_2983827168210
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#ifndef LZ4_H_2983827168210
+#define LZ4_H_2983827168210
+
 /* --- Dependency --- */
 #include    /* size_t */
 
@@ -456,9 +456,8 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInput
 LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
 LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
 
+#endif /* LZ4_H_2983827168210 */
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif /* LZ4_H_2983827168210 */
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0a625b..119dbee 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -819,13 +819,12 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
    output  : set internal values of dctx, such as
              dctxPtr->frameInfo and dctxPtr->dStage.
              Also allocates internal buffers.
-   @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize)
+   @return : nb Bytes read from src (necessarily <= srcSize)
              or an error code (testable with LZ4F_isError())
 */
 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
 {
-    BYTE FLG, BD;
-    unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID;
+    unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
     size_t frameHeaderSize;
     const BYTE* srcPtr = (const BYTE*)src;
 
@@ -852,12 +851,17 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     dctxPtr->frameInfo.frameType = LZ4F_frame;
 
     /* Flags */
-    FLG = srcPtr[4];
-    version = (FLG>>6) & _2BITS;
-    blockMode = (FLG>>5) & _1BIT;
-    blockChecksumFlag = (FLG>>4) & _1BIT;
-    contentSizeFlag = (FLG>>3) & _1BIT;
-    contentChecksumFlag = (FLG>>2) & _1BIT;
+    {   U32 const FLG = srcPtr[4];
+        U32 const version = (FLG>>6) & _2BITS;
+        U32 const blockChecksumFlag = (FLG>>4) & _1BIT;
+        blockMode = (FLG>>5) & _1BIT;
+        contentSizeFlag = (FLG>>3) & _1BIT;
+        contentChecksumFlag = (FLG>>2) & _1BIT;
+        /* validate */
+        if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+        if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong);        /* Version Number, only supported value */
+        if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
+    }
 
     /* Frame Header Size */
     frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize;
@@ -872,16 +876,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
         return srcSize;
     }
 
-    BD = srcPtr[5];
-    blockSizeID = (BD>>4) & _3BITS;
-
-    /* validate */
-    if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong);        /* Version Number, only supported value */
-    if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
-    if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
-    if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set);   /* Reserved bit */
-    if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid);    /* 4-7 only supported values for the time being */
-    if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set);  /* Reserved bits */
+    {   U32 const BD = srcPtr[5];
+        blockSizeID = (BD>>4) & _3BITS;
+        /* validate */
+        if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set);   /* Reserved bit */
+        if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid);    /* 4-7 only supported values for the time being */
+        if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set);  /* Reserved bits */
+    }
 
     /* check header */
     { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
@@ -901,11 +902,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     /* internal buffers allocation */
     {   size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
         if (bufferNeeded > dctxPtr->maxBufferSize) {   /* tmp buffers too small */
+            dctxPtr->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
             FREEMEM(dctxPtr->tmpIn);
             dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
             if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             FREEMEM(dctxPtr->tmpOutBuffer);
-            dctxPtr->maxBufferSize = 0;
             dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
             if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
             dctxPtr->maxBufferSize = bufferNeeded;
@@ -1072,7 +1073,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
 
         case dstage_getHeader:
             if ((size_t)(srcEnd-srcPtr) >= maxFHSize) {  /* enough to decode - shortcut */
-                LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);
+                LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);  /* will change dStage appropriately */
                 if (LZ4F_isError(hSize)) return hSize;
                 srcPtr += hSize;
                 break;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 7b750b6..c1acba5 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -57,15 +57,14 @@ extern "C" {
   of encoding standard metadata alongside LZ4-compressed blocks.
 */
 
-/*^***************************************************************
-*  Compiler specifics
-*****************************************************************/
-/*
-*  LZ4_DLL_EXPORT :
-*  Enable exporting of functions when building a Windows DLL
-*  LZ4FLIB_API :
-*  Control library symbols visibility.
-*/
+/*-***************************************************************
+ *  Compiler specifics
+ *****************************************************************/
+/*  LZ4_DLL_EXPORT :
+ *  Enable exporting of functions when building a Windows DLL
+ *  LZ4FLIB_API :
+ *  Control library symbols visibility.
+ */
 #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
 #  define LZ4FLIB_API __declspec(dllexport)
 #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
@@ -86,8 +85,8 @@ extern "C" {
 
 
 /*-************************************
-*  Error management
-**************************************/
+ *  Error management
+ **************************************/
 typedef size_t LZ4F_errorCode_t;
 
 LZ4FLIB_API unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
@@ -207,7 +206,7 @@ typedef struct {
   unsigned reserved[3];
 } LZ4F_compressOptions_t;
 
-/* Resource Management */
+/*---   Resource Management   ---*/
 
 #define LZ4F_VERSION 100
 LZ4FLIB_API unsigned LZ4F_getVersion(void);
@@ -223,7 +222,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr,
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 
-/* Compression */
+/*----    Compression    ----*/
 
 #define LZ4F_HEADER_SIZE_MAX 15
 /*! LZ4F_compressBegin() :
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 09a648c..c96273b 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -257,7 +257,7 @@ LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data,
  *  Result is provided in 2 parts :
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
- * `sourceSizePtr` : value will be updated to indicate how much bytes were read from `source`
+ * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
  * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
  *             beyond that level, compression performance will be much reduced due to internal incompatibilities
  */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a36d20b..2e22912 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -664,7 +664,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
             LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
             blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
-            DISPLAYLEVEL(3, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
+            DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
             FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
             FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
-- 
cgit v0.12


From eff6166eb84a7aca764cf310bc9eef23d003c354 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 15:51:08 -0700
Subject: minor price function optimization

---
 lib/lz4opt.h | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 582aa14..7e2e0f6 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -36,14 +36,12 @@
 #define LZ4_OPT_NUM   (1<<12)
 
 
-typedef struct
-{
+typedef struct {
     int off;
     int len;
 } LZ4HC_match_t;
 
-typedef struct
-{
+typedef struct {
     int price;
     int off;
     int mlen;
@@ -54,8 +52,8 @@ typedef struct
 /* price in bits */
 FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 {
-    size_t price = 8*litlen;
-    if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255);
+    size_t price = litlen;
+    if (litlen >= (size_t)RUN_MASK) price += 1+(litlen-RUN_MASK)/255;
     return price;
 }
 
@@ -63,12 +61,12 @@ FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 /* requires mlen >= MINMATCH */
 FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
 {
-    size_t price = 16 + 8; /* 16-bit offset + token */
+    size_t price = 2 + 1; /* 16-bit offset + token */
 
     price += LZ4HC_literalsPrice(litlen);
 
     mlen -= MINMATCH;
-    if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255);
+    if (mlen >= (size_t)ML_MASK) price+= 1+(mlen-ML_MASK)/255;
 
     return price;
 }
-- 
cgit v0.12


From 118b47f3d1c7bc8898811d47a127b6af9743e2b5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 16:53:35 -0700
Subject: improved lz4opt speed (~4%)

---
 lib/lz4opt.h | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 7e2e0f6..b74c946 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -65,8 +65,8 @@ FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
 
     price += LZ4HC_literalsPrice(litlen);
 
-    mlen -= MINMATCH;
-    if (mlen >= (size_t)ML_MASK) price+= 1+(mlen-ML_MASK)/255;
+    if (mlen >= (size_t)(ML_MASK+MINMATCH))
+        price+= 1+(mlen-(ML_MASK+MINMATCH))/255;
 
     return price;
 }
@@ -190,12 +190,12 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
 }
 
 
-#define SET_PRICE(pos, mlen, offset, litlen, price)    \
+#define SET_PRICE(pos, mlen, offset, ll, price)        \
 {                                                      \
     while (last_pos < pos)  { opt[last_pos+1].price = 1<<30; last_pos++; } \
     opt[pos].mlen = (int)mlen;                         \
     opt[pos].off = (int)offset;                        \
-    opt[pos].litlen = (int)litlen;                     \
+    opt[pos].litlen = (int)ll;                         \
     opt[pos].price = (int)price;                       \
 }
 
@@ -207,7 +207,7 @@ static int LZ4HC_compress_optimal (
     int inputSize,
     int maxOutputSize,
     limitedOutput_directive limit,
-    const size_t sufficient_len,
+    size_t sufficient_len,
     const int fullUpdate
     )
 {
@@ -226,6 +226,7 @@ static int LZ4HC_compress_optimal (
     BYTE* const oend = op + maxOutputSize;
 
     /* init */
+    if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
     ctx->end += inputSize;
     ip++;
 
@@ -247,14 +248,13 @@ static int LZ4HC_compress_optimal (
 
         /* set prices using matches at position = 0 */
         for (i = 0; i < match_num; i++) {
-           mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
-           best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
-           while (mlen <= best_mlen) {
-                litlen = 0;
-                price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
-                SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
+            mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
+            best_mlen = matches[i].len;   /* necessarily < LZ4_OPT_NUM */
+            while (mlen <= best_mlen) {
+                price = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
+                SET_PRICE(mlen, mlen, matches[i].off, 0, price);   /* updates last_pos and opt[pos] */
                 mlen++;
-           }
+            }
         }
 
         if (last_pos < MINMATCH) { ip++; continue; }
-- 
cgit v0.12


From 8bd32a17b112dd46f61d835e4368ff1bc4464048 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 17:42:47 -0700
Subject: made SET_PRICE macro more usable

previous version would use argument to also change target member.
Now, only values are transferred
---
 lib/lz4opt.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index b74c946..f12468a 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -190,13 +190,13 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
 }
 
 
-#define SET_PRICE(pos, mlen, offset, ll, price)        \
+#define SET_PRICE(pos, ml, offset, ll, cost)           \
 {                                                      \
     while (last_pos < pos)  { opt[last_pos+1].price = 1<<30; last_pos++; } \
-    opt[pos].mlen = (int)mlen;                         \
+    opt[pos].mlen = (int)ml;                           \
     opt[pos].off = (int)offset;                        \
     opt[pos].litlen = (int)ll;                         \
-    opt[pos].price = (int)price;                       \
+    opt[pos].price = (int)cost;                        \
 }
 
 
@@ -211,7 +211,7 @@ static int LZ4HC_compress_optimal (
     const int fullUpdate
     )
 {
-    LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
+    LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];   /* this uses a bit too much stack memory to my taste ... */
     LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
     const BYTE *inr = NULL;
     size_t res, cur, cur2;
-- 
cgit v0.12


From f513020a6df8e1c8c51e03d849c8574f821b765b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 18:07:53 -0700
Subject: slight btopt speed improvement

removing a useless test
---
 lib/lz4opt.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index f12468a..155b1fc 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -278,8 +278,8 @@ static int LZ4HC_compress_optimal (
 
             mlen = 1;
             best_mlen = 0;
-            if (cur > last_pos || price < (size_t)opt[cur].price)
-                SET_PRICE(cur, mlen, best_mlen, litlen, price);
+            if (price < (size_t)opt[cur].price)
+                SET_PRICE(cur, mlen, best_mlen, litlen, price);   /* note : increases last_pos */
 
             if (cur == last_pos || inr >= mflimit) break;
 
-- 
cgit v0.12


From cea8f60df14d3c538e0a420eb15b808159ab6c09 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 17 Mar 2017 18:07:53 -0700
Subject: slight btopt speed improvement

removing a useless test
---
 lib/lz4opt.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index f12468a..36c84bb 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -278,13 +278,14 @@ static int LZ4HC_compress_optimal (
 
             mlen = 1;
             best_mlen = 0;
-            if (cur > last_pos || price < (size_t)opt[cur].price)
-                SET_PRICE(cur, mlen, best_mlen, litlen, price);
+            if (price < (size_t)opt[cur].price)
+                SET_PRICE(cur, mlen, best_mlen, litlen, price);   /* note : increases last_pos */
 
             if (cur == last_pos || inr >= mflimit) break;
 
             match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
+            if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) {
+                /* immediate encoding */
                 best_mlen = matches[match_num-1].len;
                 best_off = matches[match_num-1].off;
                 last_pos = cur + 1;
-- 
cgit v0.12


From b8bc70022bb5b9f1775796c3df90dac5d2f55686 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 20 Mar 2017 02:57:41 -0700
Subject: minor refactor

---
 lib/lz4hc.c  |   4 ++
 lib/lz4opt.h | 143 +++++++++++++++++++++++++++++------------------------------
 2 files changed, 75 insertions(+), 72 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f3c9946..03a0301 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -243,6 +243,10 @@ typedef enum {
 static unsigned debug = 0;
 #endif
 
+
+/* LZ4HC_encodeSequence() :
+ * @return : 0 if ok,
+ *           1 if buffer issue detected */
 FORCE_INLINE int LZ4HC_encodeSequence (
     const BYTE** ip,
     BYTE** op,
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 36c84bb..d51d491 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -53,7 +53,7 @@ typedef struct {
 FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 {
     size_t price = litlen;
-    if (litlen >= (size_t)RUN_MASK) price += 1+(litlen-RUN_MASK)/255;
+    if (litlen >= (size_t)RUN_MASK) price += 1 + (litlen-RUN_MASK)/255;
     return price;
 }
 
@@ -213,9 +213,6 @@ static int LZ4HC_compress_optimal (
 {
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];   /* this uses a bit too much stack memory to my taste ... */
     LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
-    const BYTE *inr = NULL;
-    size_t res, cur, cur2;
-    size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -232,13 +229,16 @@ static int LZ4HC_compress_optimal (
 
     /* Main Loop */
     while (ip < mflimit) {
+        size_t const llen = ip - anchor;
+        size_t last_pos = 0;
+        size_t match_num, cur, best_mlen, best_off;
         memset(opt, 0, sizeof(LZ4HC_optimal_t));
-        last_pos = 0;
-        llen = ip - anchor;
+
         match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!match_num) { ip++; continue; }
 
         if ((size_t)matches[match_num-1].len > sufficient_len) {
+            /* good enough solution : immediate encoding */
             best_mlen = matches[match_num-1].len;
             best_off = matches[match_num-1].off;
             cur = 0;
@@ -247,43 +247,45 @@ static int LZ4HC_compress_optimal (
         }
 
         /* set prices using matches at position = 0 */
-        for (i = 0; i < match_num; i++) {
-            mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
-            best_mlen = matches[i].len;   /* necessarily < LZ4_OPT_NUM */
-            while (mlen <= best_mlen) {
-                price = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
-                SET_PRICE(mlen, mlen, matches[i].off, 0, price);   /* updates last_pos and opt[pos] */
-                mlen++;
-            }
-        }
+        {   size_t matchNb;
+            for (matchNb = 0; matchNb < match_num; matchNb++) {
+                size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
+                best_mlen = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
+                for ( ; mlen <= best_mlen ; mlen++) {
+                    size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
+                    SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_pos and opt[pos] */
+        }   }   }
 
-        if (last_pos < MINMATCH) { ip++; continue; }
+        if (last_pos < MINMATCH) { ip++; continue; }  /* note : on clang at least, this test improves performance */
 
         /* check further positions */
         opt[0].mlen = opt[1].mlen = 1;
         for (cur = 1; cur <= last_pos; cur++) {
-            inr = ip + cur;
-
-            if (opt[cur-1].mlen == 1) {
-                litlen = opt[cur-1].litlen + 1;
-                if (cur != litlen) {
-                    price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
+            const BYTE* const curPtr = ip + cur;
+
+            /* establish baseline price if cur is literal */
+            {   size_t price, litlen;
+                if (opt[cur-1].mlen == 1) {
+                    /* no match at previous position */
+                    litlen = opt[cur-1].litlen + 1;
+                    if (cur > litlen) {
+                        price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
+                    } else {
+                        price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
+                    }
                 } else {
-                    price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
+                    litlen = 1;
+                    price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
                 }
-            } else {
-                litlen = 1;
-                price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
-            }
 
-            mlen = 1;
-            best_mlen = 0;
-            if (price < (size_t)opt[cur].price)
-                SET_PRICE(cur, mlen, best_mlen, litlen, price);   /* note : increases last_pos */
+                best_mlen = 0;
+                if (price < (size_t)opt[cur].price)
+                    SET_PRICE(cur, 1, 0, litlen, price);   /* note : increases last_pos */
+            }
 
-            if (cur == last_pos || inr >= mflimit) break;
+            if (cur == last_pos || curPtr >= mflimit) break;
 
-            match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) {
                 /* immediate encoding */
                 best_mlen = matches[match_num-1].len;
@@ -293,60 +295,57 @@ static int LZ4HC_compress_optimal (
             }
 
             /* set prices using matches at position = cur */
-            for (i = 0; i < match_num; i++) {
-                mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
-                cur2 = cur;
-                best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
-
-                while (mlen <= best_mlen) {
-                    if (opt[cur2].mlen == 1) {
-                        litlen = opt[cur2].litlen;
-
-                        if (cur2 != litlen)
-                            price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen);
-                        else
-                            price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
-                    } else {
-                        litlen = 0;
-                        price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
-                    }
-
-                    if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
-                        SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
-                    }
-                    mlen++;
-                }
-            }
+            {   size_t matchNb;
+                for (matchNb = 0; matchNb < match_num; matchNb++) {
+                    size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
+                    best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
+                                (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur;
+
+                    for ( ; ml <= best_mlen ; ml++) {
+                        size_t ll, price;
+                        if (opt[cur].mlen == 1) {
+                            ll = opt[cur].litlen;
+                            if (cur > ll)
+                                price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml);
+                            else
+                                price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen);
+                        } else {
+                            ll = 0;
+                            price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
+                        }
+
+                        if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) {
+                            SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price);
+            }   }   }   }
         } /* for (cur = 1; cur <= last_pos; cur++) */
 
         best_mlen = opt[last_pos].mlen;
         best_off = opt[last_pos].off;
         cur = last_pos - best_mlen;
 
-encode: /* cur, last_pos, best_mlen, best_off have to be set */
+encode: /* cur, last_pos, best_mlen, best_off must be set */
         opt[0].mlen = 1;
-        while (1) {
-            mlen = opt[cur].mlen;
-            offset = opt[cur].off;
+        while (1) {  /* from end to beginning */
+            size_t const ml = opt[cur].mlen;
+            int const offset = opt[cur].off;
             opt[cur].mlen = (int)best_mlen;
             opt[cur].off = (int)best_off;
-            best_mlen = mlen;
+            best_mlen = ml;
             best_off = offset;
-            if (mlen > cur) break;
-            cur -= mlen;
+            if (ml > cur) break;
+            cur -= ml;
         }
 
+        /* encode all recorded sequences */
         cur = 0;
         while (cur < last_pos) {
-            mlen = opt[cur].mlen;
-            if (mlen == 1) { ip++; cur++; continue; }
-            offset = opt[cur].off;
-            cur += mlen;
-
-            res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
-            if (res) return 0;
+            int const ml = opt[cur].mlen;
+            int const offset = opt[cur].off;
+            if (ml == 1) { ip++; cur++; continue; }
+            cur += ml;
+            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0;
         }
-    }
+    }  /* while (ip < mflimit) */
 
     /* Encode Last Literals */
     {   int lastRun = (int)(iend - anchor);
-- 
cgit v0.12


From b1daffc4e5653f05fa6a7eb40d0751c300837636 Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Thu, 23 Mar 2017 03:48:51 +0000
Subject: cli: add GNU separator -- specifying that all following arguments are
 files

This option is supported by other compressors with compatible cli,
so add it to lz4 as well for better compatibility.
---
 NEWS              | 1 +
 programs/lz4.1.md | 3 +++
 programs/lz4cli.c | 4 +++-
 tests/Makefile    | 8 ++++++++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 742217f..4fb8fdd 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 v1.7.6
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
+cli : added GNU separator -- specifying that all following arguments are files
 API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : dragonFlyBSD, OpenBSD, NetBSD supported
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 48f3152..6f42904 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -194,6 +194,9 @@ only the latest one will be applied.
 * `--rm` :
   Delete source files on successful compression or decompression
 
+* `--` :
+  Treat all subsequent arguments as files
+
 
 ### Benchmark mode
 
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 0578bde..7a75f55 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -271,6 +271,7 @@ int main(int argc, const char** argv)
         forceStdout=0,
         main_pause=0,
         multiple_inputs=0,
+        all_arguments_are_files=0,
         operationResult=0;
     operationMode_e mode = om_auto;
     const char* input_filename = NULL;
@@ -315,7 +316,7 @@ int main(int argc, const char** argv)
         if(!argument) continue;   /* Protection if argument empty */
 
         /* Short commands (note : aggregated short commands are allowed) */
-        if (argument[0]=='-') {
+        if (!all_arguments_are_files && argument[0]=='-') {
             /* '-' means stdin/stdout */
             if (argument[1]==0) {
                 if (!input_filename) input_filename=stdinmark;
@@ -325,6 +326,7 @@ int main(int argc, const char** argv)
 
             /* long commands (--long-word) */
             if (argument[1]=='-') {
+                if (!strcmp(argument,  "--")) { all_arguments_are_files = 1; continue; }
                 if (!strcmp(argument,  "--compress")) { mode = om_compress; continue; }
                 if ((!strcmp(argument, "--decompress"))
                     || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; }
diff --git a/tests/Makefile b/tests/Makefile
index d68c700..bc22423 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -252,6 +252,14 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	$(DIFF) -q tmp tmp3
 	$(PRGDIR)/lz4cat < tmp > tmp2      # checks lz4cat works with stdin (#285)
 	$(DIFF) -q tmp tmp2
+	cp tmp ./-d
+	$(LZ4) --rm -- -d -d.lz4           # compresses ./d into ./-d.lz4
+	test   -f ./-d.lz4
+	test ! -f ./-d
+	mv ./-d.lz4 ./-z
+	$(LZ4) -d --rm -- -z tmp4          # uncompresses ./-z into tmp4
+	test ! -f ./-z
+	$(DIFF) -q tmp tmp4
 	@$(RM) tmp*
 
 test-lz4-hugefile: lz4 datagen
-- 
cgit v0.12


From dab3590fce7322fafb97db34690ec24bf0334ab0 Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Fri, 24 Mar 2017 03:12:48 +0000
Subject: Export deprecated symbols declared in lz4hc.h

Commit 25b243588585b06a2d947372284cbfe00da930e9 exported deprecated
symbols declared in lz4.h by marking all LZ4_DEPRECATED functions with
LZ4LIB_API attribute.  This change does the same with functions declared
in lz4hc.h file, thus extending the export to lz4hc.h.

As result, the following 17 deprecated functions are exported again:

LZ4_compressHC
LZ4_compressHC2
LZ4_compressHC2_continue
LZ4_compressHC2_limitedOutput
LZ4_compressHC2_limitedOutput_continue
LZ4_compressHC2_limitedOutput_withStateHC
LZ4_compressHC2_withStateHC
LZ4_compressHC_continue
LZ4_compressHC_limitedOutput
LZ4_compressHC_limitedOutput_continue
LZ4_compressHC_limitedOutput_withStateHC
LZ4_compressHC_withStateHC
LZ4_createHC
LZ4_freeHC
LZ4_resetStreamStateHC
LZ4_sizeofStreamStateHC
LZ4_slideInputBufferHC
---
 lib/lz4hc.h | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index c96273b..2e3880d 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -198,25 +198,25 @@ union LZ4_streamHC_u {
 
 /* deprecated compression functions */
 /* these functions will trigger warning messages in future releases */
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC               (const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC               (const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
 
 /* Deprecated Streaming functions using older model; should no longer be used */
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDictHC() instead")     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_freeStreamHC() instead")   int   LZ4_freeHC (void* LZ4HC_Data);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
-LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
-LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead")     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead")   int   LZ4_freeHC (void* LZ4HC_Data);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
+LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
 
 
 #if defined (__cplusplus)
-- 
cgit v0.12


From 0863931a1c65c5f3caf7cf83c8780ac8e85d091e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 24 Mar 2017 10:34:18 -0700
Subject: fixed API comment for LZ4F_createCompressionContext()

as reported by @nh2
---
 lib/lz4frame.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index c1acba5..a0cf0ab 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -212,7 +212,7 @@ typedef struct {
 LZ4FLIB_API unsigned LZ4F_getVersion(void);
 /*! LZ4F_createCompressionContext() :
  * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
  * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
  * The function will provide a pointer to a fully allocated LZ4F_cctx object.
  * If @return != zero, there was an error during context creation.
-- 
cgit v0.12


From 60b6d2907ffe1b07429781df9323d7636d62a1bd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 24 Mar 2017 11:29:35 -0700
Subject: improved level 10 speed for degenerated cases

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 03a0301..ac15d20 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -550,7 +550,7 @@ static int LZ4HC_compress_generic (
         if (limit == limitedDestSize) cLevel = 10;
         switch (cLevel) {
             case 10:
-                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (16-1), limit);
+                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (15-1), limit);
             case 11:
                 ctx->searchNum = LZ4HC_getSearchNum(cLevel);
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
-- 
cgit v0.12


From e9c3b14f29486feee823f95b238fe0a83fb64670 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Fri, 24 Mar 2017 16:33:14 -0700
Subject: Ignore extensions in exe name matching

---
 programs/lz4cli.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 0578bde..da02b81 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -242,10 +242,19 @@ static void waitEnter(void)
 
 static const char* lastNameFromPath(const char* path)
 {
-    const char* name = strrchr(path, '/');
-    if (name==NULL) name = strrchr(path, '\\');   /* windows */
-    if (name==NULL) return path;
-    return name+1;
+    const char* name = path;
+    if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
+    if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
+    return name;
+}
+
+/*! exeNameMatch() :
+    @return : a non-zero value if exeName matches test, excluding the extension
+   */
+static int exeNameMatch(const char* exeName, const char* test)
+{
+    return !strncmp(exeName, test, strlen(test)) &&
+        (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
 }
 
 /*! readU32FromChar() :
@@ -297,7 +306,7 @@ int main(int argc, const char** argv)
     LZ4IO_setOverwrite(0);
 
     /* lz4cat predefined behavior */
-    if (!strcmp(exeName, LZ4CAT)) {
+    if (exeNameMatch(exeName, LZ4CAT)) {
         mode = om_decompress;
         LZ4IO_setOverwrite(1);
         LZ4IO_setRemoveSrcFile(0);
@@ -306,7 +315,7 @@ int main(int argc, const char** argv)
         displayLevel=1;
         multiple_inputs=1;
     }
-    if (!strcmp(exeName, UNLZ4)) { mode = om_decompress; }
+    if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; }
 
     /* command switches */
     for(i=1; i
Date: Mon, 27 Mar 2017 12:10:10 -0700
Subject: Fix IS_CONSOLE returning 1 for NUL on windows

---
 programs/platform.h | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/programs/platform.h b/programs/platform.h
index 51ce1ac..1238140 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -108,9 +108,18 @@ extern "C" {
 #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
 #  include    /* isatty */
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__)
 #  include        /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#elif defined(WIN32) || defined(_WIN32)
+#  include       /* _isatty */
+#  include  /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+#  include    /* FILE */
+static inline int IS_CONSOLE(FILE* stdStream)
+{
+    DWORD dummy;
+    return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
+}
 #else
 #  define IS_CONSOLE(stdStream) 0
 #endif
-- 
cgit v0.12


From 9b1f00056aaa8b1c4d0877d1f73c9da6c2b1e2d7 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 27 Mar 2017 12:26:04 -0700
Subject: Fix inline compile errors

---
 programs/platform.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/platform.h b/programs/platform.h
index 1238140..66491b6 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -115,7 +115,7 @@ extern "C" {
 #  include       /* _isatty */
 #  include  /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
 #  include    /* FILE */
-static inline int IS_CONSOLE(FILE* stdStream)
+static __inline int IS_CONSOLE(FILE* stdStream)
 {
     DWORD dummy;
     return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
-- 
cgit v0.12


From f0a7651fce53f5e85da6140f9d075b730ae6eac7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Mar 2017 17:10:01 -0700
Subject: Safer LZ4_getFrameInfo()

LZ4_getFrameInfo() is now guaranteed to keep dctx state clean,
even in case of failure.
---
 lib/lz4frame.c        | 112 +++++++++++++++++++++++++++++---------------------
 lib/lz4frame.h        |  27 +++++++-----
 lib/lz4frame_static.h |  35 +++++++++-------
 tests/frametest.c     |  26 ++++++++----
 4 files changed, 120 insertions(+), 80 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 119dbee..518194d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -779,7 +779,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr)
 
 /*==---   Streaming Decompression operations   ---==*/
 
-typedef enum { dstage_getHeader=0, dstage_storeHeader,
+typedef enum {
+    dstage_getHeader=0, dstage_storeHeader,
+    dstage_init,
     dstage_getCBlockSize, dstage_storeCBlockSize,
     dstage_copyDirect,
     dstage_getCBlock, dstage_storeCBlock,
@@ -896,65 +898,57 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     if (contentSizeFlag)
         dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
 
-    /* init */
-    if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
-
-    /* internal buffers allocation */
-    {   size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
-        if (bufferNeeded > dctxPtr->maxBufferSize) {   /* tmp buffers too small */
-            dctxPtr->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
-            FREEMEM(dctxPtr->tmpIn);
-            dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
-            if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            FREEMEM(dctxPtr->tmpOutBuffer);
-            dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
-            if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            dctxPtr->maxBufferSize = bufferNeeded;
-    }   }
-    dctxPtr->tmpInSize = 0;
-    dctxPtr->tmpInTarget = 0;
-    dctxPtr->dict = dctxPtr->tmpOutBuffer;
-    dctxPtr->dictSize = 0;
-    dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
-    dctxPtr->tmpOutStart = 0;
-    dctxPtr->tmpOutSize = 0;
-
-    dctxPtr->dStage = dstage_getCBlockSize;
+    dctxPtr->dStage = dstage_init;
 
     return frameHeaderSize;
 }
 
 
 /*! LZ4F_getFrameInfo() :
-*   Decodes frame header information, such as blockSize. Usage is optional.
-*   The objective is to extract header information before receiving decompressed data, typically for allocation purposes.
-*   LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t.
-*   The number of bytes consumed from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
-*   Decompression must resume from where it stopped (srcBuffer + *srcSizePtr)
-*   @return : hint of the better `srcSize` to use for next call to LZ4F_decompress,
-*             or an error code which can be tested using LZ4F_isError().
-*/
+ * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
+ * Its usage is optional. The objective is to provide relevant information for allocation purposes.
+ * This function works in 2 situations :
+ *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+ *     Amount of input data provided must be large enough to successfully decode the frame header.
+ *     A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
+ *   - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
+ * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
+ * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
+ *           or an error code which can be tested using LZ4F_isError()
+ * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ */
 LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr,
                                    const void* srcBuffer, size_t* srcSizePtr)
 {
-    if (dctxPtr->dStage > dstage_storeHeader) {  /* note : requires dstage_* header related to be at beginning of enum */
+    if (dctxPtr->dStage > dstage_storeHeader) {  /* assumption :  dstage_* header enum at beginning of range */
         /* frameInfo already decoded */
         size_t o=0, i=0;
         *srcSizePtr = 0;
         *frameInfoPtr = dctxPtr->frameInfo;
         return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);  /* returns : recommended nb of bytes for LZ4F_decompress() */
     } else {
-        size_t nextSrcSize, o=0;
-        size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
-        if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
-        if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
-
-        *srcSizePtr = hSize;
-        nextSrcSize = LZ4F_decompress(dctxPtr, NULL, &o, srcBuffer, srcSizePtr, NULL);
-        if (dctxPtr->dStage <= dstage_storeHeader) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* should not happen, already checked */
-        *frameInfoPtr = dctxPtr->frameInfo;
-        return nextSrcSize;
-    }
+        if (dctxPtr->dStage == dstage_storeHeader) {
+            /* frame decoding already started, in the middle of header => automatic fail */
+            *srcSizePtr = 0;
+            return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
+        } else {
+            size_t decodeResult;
+            size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
+            if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
+            if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
+
+            decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
+            if (LZ4F_isError(decodeResult)) {
+                *srcSizePtr = 0;
+            } else {
+                *srcSizePtr = decodeResult;
+                decodeResult = BHSize;   /* block header size */
+            }
+            *frameInfoPtr = dctxPtr->frameInfo;
+            return decodeResult;
+    }   }
 }
 
 
@@ -1064,7 +1058,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
     *srcSizePtr = 0;
     *dstSizePtr = 0;
 
-    /* programmed as a state machine */
+    /* behaves like a state machine */
 
     while (doAnotherStage) {
 
@@ -1079,6 +1073,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 break;
             }
             dctxPtr->tmpInSize = 0;
+            if (srcEnd-srcPtr == 0) return minFHSize;   /* 0-size input */
             dctxPtr->tmpInTarget = minFHSize;   /* minimum to attempt decode */
             dctxPtr->dStage = dstage_storeHeader;
             /* pass-through */
@@ -1100,6 +1095,31 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 break;
             }
 
+        case dstage_init:
+            if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
+            /* internal buffers allocation */
+            {   size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
+                if (bufferNeeded > dctxPtr->maxBufferSize) {   /* tmp buffers too small */
+                    dctxPtr->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
+                    FREEMEM(dctxPtr->tmpIn);
+                    dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
+                    if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    FREEMEM(dctxPtr->tmpOutBuffer);
+                    dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+                    if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    dctxPtr->maxBufferSize = bufferNeeded;
+            }   }
+            dctxPtr->tmpInSize = 0;
+            dctxPtr->tmpInTarget = 0;
+            dctxPtr->dict = dctxPtr->tmpOutBuffer;
+            dctxPtr->dictSize = 0;
+            dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
+            dctxPtr->tmpOutStart = 0;
+            dctxPtr->tmpOutSize = 0;
+
+            dctxPtr->dStage = dstage_getCBlockSize;
+            /* pass-through */
+
         case dstage_getCBlockSize:
             if ((size_t)(srcEnd - srcPtr) >= BHSize) {
                 selectedIn = srcPtr;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index a0cf0ab..7c33464 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -303,25 +303,30 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
 
 
-/* ======   Decompression   ======*/
+/*-***********************************
+*  Streaming decompression functions
+*************************************/
 
-/*!LZ4F_getFrameInfo() :
- * This function decodes frame header information (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
- * A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
+/*! LZ4F_getFrameInfo() :
+ * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
+ * Its usage is optional. The objective is to provide relevant information for allocation purposes.
+ * This function works in 2 situations :
+ *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+ *     Amount of input data provided must be large enough to successfully decode the frame header.
+ *     A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
+ *   - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
  * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from this point (srcBuffer + *srcSizePtr).
- * Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
- * Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
+ * Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
  *           or an error code which can be tested using LZ4F_isError()
- *           (typically, when there is not enough src bytes to fully decode the frame header)
+ * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  */
 LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
-/*!LZ4F_decompress() :
+/*! LZ4F_decompress() :
  * Call this function repetitively to regenerate data compressed within `srcBuffer`.
  * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
  *
@@ -337,7 +342,7 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
  *
  * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
  * This is just a hint though, it's always possible to provide any srcSize.
  * When a frame is fully decoded, @return will be 0 (no more data expected).
  * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index f2228a5..d3bae82 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -43,7 +43,7 @@ extern "C" {
 /* lz4frame_static.h should be used solely in the context of static linking.
  * It contains definitions which are not stable and may change in the future.
  * Never use it in the context of DLL linking.
- * */
+ */
 
 
 /* ---   Dependency   --- */
@@ -52,25 +52,32 @@ extern "C" {
 
 /* ---   Error List   --- */
 #define LZ4F_LIST_ERRORS(ITEM) \
-        ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
-        ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(OK_NoError) \
+        ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) \
+        ITEM(ERROR_blockMode_invalid) \
+        ITEM(ERROR_contentChecksumFlag_invalid) \
         ITEM(ERROR_compressionLevel_invalid) \
-        ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
+        ITEM(ERROR_headerVersion_wrong) \
+        ITEM(ERROR_blockChecksum_unsupported) \
+        ITEM(ERROR_reservedFlag_set) \
         ITEM(ERROR_allocation_failed) \
-        ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
-        ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_srcSize_tooLarge) \
+        ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameHeader_incomplete) \
+        ITEM(ERROR_frameType_unknown) \
+        ITEM(ERROR_frameSize_wrong) \
         ITEM(ERROR_srcPtr_wrong) \
         ITEM(ERROR_decompressionFailed) \
-        ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_headerChecksum_invalid) \
+        ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_frameDecoding_alreadyStarted) \
         ITEM(ERROR_maxCode)
 
-#define LZ4F_DISABLE_OLD_ENUMS   /* comment to enable deprecated enums */
-#ifndef LZ4F_DISABLE_OLD_ENUMS
-#  define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
-#else
-#  define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
-#endif
-typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;  /* enum is exposed, to handle specific errors; compare function result to -enum value */
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+
+/* enum list is exposed, to handle specific errors */
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
 
 LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
 
diff --git a/tests/frametest.c b/tests/frametest.c
index e2e0f86..f7e3abf 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -276,17 +276,25 @@ int basicTests(U32 seed, double compressibility)
             if (LZ4F_isError(errorCode)) goto _output_error;
             DISPLAYLEVEL(3, " %u  \n", (unsigned)errorCode);
 
-            DISPLAYLEVEL(3, "get FrameInfo on null input : ");
-            errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
-            if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
-            DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
+            DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
+            {   size_t nullSize = 0;
+                size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
+                if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
+                    DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
+                    goto _output_error;
+                }
+                DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
+            }
 
             DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
-            iSize = 6;
-            errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
-            if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
-            DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
-            ip += iSize;
+            {   size_t inputSize = 6;
+                size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
+                if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
+                    DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
+                    goto _output_error;
+                }
+                DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
+            }
 
             DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
             iSize = 15 - iSize;
-- 
cgit v0.12


From fc31257ab2efd2e2d4d1aab97dcde8622ab6e92a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Mar 2017 17:36:12 -0700
Subject: added LZ4F_resetDecompressionContext()

---
 tests/frametest.c | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index f7e3abf..ae8416b 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -195,7 +195,7 @@ int basicTests(U32 seed, double compressibility)
 
     /* Special case : null-content frame */
     testSize = 0;
-    DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n");
+    DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
     if (LZ4F_isError(cSize)) goto _output_error;
     DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
@@ -218,7 +218,7 @@ int basicTests(U32 seed, double compressibility)
 
     /* test one-pass frame compression */
     testSize = COMPRESSIBLE_NOISE_LENGTH;
-    DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n");
+    DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
     if (LZ4F_isError(cSize)) goto _output_error;
     DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
@@ -232,10 +232,10 @@ int basicTests(U32 seed, double compressibility)
         LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
         if (LZ4F_isError(errorCode)) goto _output_error;
 
-        DISPLAYLEVEL(3, "Single Pass decompression : \n");
+        DISPLAYLEVEL(3, "Single Pass decompression : ");
         { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
           if (LZ4F_isError(decompressError)) goto _output_error; }
-        { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+        { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
           if (crcDest != crcOrig) goto _output_error; }
         DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
 
@@ -286,7 +286,7 @@ int basicTests(U32 seed, double compressibility)
                 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
             }
 
-            DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
+            DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : ");
             {   size_t inputSize = 6;
                 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
                 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
@@ -296,7 +296,7 @@ int basicTests(U32 seed, double compressibility)
                 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
             }
 
-            DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
+            DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
             iSize = 15 - iSize;
             errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
             if (LZ4F_isError(errorCode)) goto _output_error;
@@ -304,7 +304,7 @@ int basicTests(U32 seed, double compressibility)
             ip += iSize;
         }
 
-        DISPLAYLEVEL(3, "Byte after byte : \n");
+        DISPLAYLEVEL(3, "Byte after byte : ");
         {   BYTE* const ostart = (BYTE*)decodedBuffer;
             BYTE* op = ostart;
             BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
@@ -326,20 +326,20 @@ int basicTests(U32 seed, double compressibility)
         dCtx = NULL;
     }
 
-    DISPLAYLEVEL(3, "Using 64 KB block : \n");
+    DISPLAYLEVEL(3, "Using 64 KB block : ");
     prefs.frameInfo.blockSizeID = LZ4F_max64KB;
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
     if (LZ4F_isError(cSize)) goto _output_error;
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
-    DISPLAYLEVEL(3, "without checksum : \n");
+    DISPLAYLEVEL(3, "without checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
     if (LZ4F_isError(cSize)) goto _output_error;
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
-    DISPLAYLEVEL(3, "Using 256 KB block : \n");
+    DISPLAYLEVEL(3, "Using 256 KB block : ");
     prefs.frameInfo.blockSizeID = LZ4F_max256KB;
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
@@ -358,7 +358,7 @@ int basicTests(U32 seed, double compressibility)
         { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
           if (LZ4F_isError(createError)) goto _output_error; }
 
-        DISPLAYLEVEL(3, "random segment sizes : \n");
+        DISPLAYLEVEL(3, "random segment sizes : ");
         while (ip < iend) {
             unsigned const nbBits = FUZ_rand(&randState) % maxBits;
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
-- 
cgit v0.12


From b88df6b1b058a1a4884e8c7c80ce8b7e0839eb30 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 Mar 2017 12:51:08 -0700
Subject: Improved comments on LZ4F_getFrameInfo()

and added LZ4F_resetCompressionContext()
---
 lib/lz4frame.c        |  6 ++++++
 lib/lz4frame.h        | 14 +++++++++-----
 lib/lz4frame_static.h |  9 +++++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 518194d..1184a0f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -792,6 +792,12 @@ typedef enum {
     dstage_skipSkippable
 } dStage_t;
 
+LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
+{
+    dctx->dStage = dstage_getHeader;
+    return 0;
+}
+
 
 /*! LZ4F_headerSize() :
 *   @return : size of frame header
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 7c33464..76b4e69 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -300,7 +300,7 @@ typedef struct {
  * That is, it should be == 0 if decompression has been completed fully and correctly.
  */
 LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
 
 /*-***********************************
@@ -309,12 +309,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx
 
 /*! LZ4F_getFrameInfo() :
  * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. The objective is to provide relevant information for allocation purposes.
+ * Its usage is optional. Extracted information can be useful for allocation purposes, typically.
  * This function works in 2 situations :
  *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
- *     Amount of input data provided must be large enough to successfully decode the frame header.
- *     A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
- *   - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
+ *     Input size must be large enough to successfully decode the entire frame header.
+ *     Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
+ *     It's allowed to provide more input data than this minimum.
+ *   - After decoding has been started.
+ *     In which case, no input is read, frame parameters are extracted from dctx.
+ *     If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
  * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
  * Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
@@ -348,6 +351,7 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
  * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
  *
  * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  */
 LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    void* dstBuffer, size_t* dstSizePtr,
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index d3bae82..8ea496d 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -50,6 +50,15 @@ extern "C" {
 #include "lz4frame.h"
 
 
+/* ---   Experimental functions   --- */
+/* LZ4F_resetDecompressionContext() :
+ * LZ4F_decompress() does not guarantee to leave dctx in clean state in case of errors.
+ * In order to re-use a dctx after a decompression error,
+ * use LZ4F_resetDecompressionContext() first.
+ * dctx will be able to start decompression on a new frame */
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+
+
 /* ---   Error List   --- */
 #define LZ4F_LIST_ERRORS(ITEM) \
         ITEM(OK_NoError) \
-- 
cgit v0.12


From 6226d52eabf2ef71cdef13827fda96c71707a238 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 29 Mar 2017 14:18:10 -0700
Subject: fixed minor Visual warning

---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 1184a0f..e04fe83 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -764,7 +764,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionC
     return LZ4F_OK_NoError;
 }
 
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctxPtr)
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr)
 {
     LZ4F_errorCode_t result = LZ4F_OK_NoError;
     if (dctxPtr != NULL) {   /* can accept NULL input, like free() */
-- 
cgit v0.12


From e2827775ee80d2ef985858727575df31fc60f1f3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 30 Mar 2017 12:22:17 -0700
Subject: make __packed memory access default for gcc

It's always as good or better then memcpy()
but depends on gcc-specific extension.

solves https://github.com/facebook/zstd/issues/620
---
 lib/lz4.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 4558117..c9c5a07 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -63,16 +63,15 @@
  * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
  * Method 2 : direct access. This method is portable but violate C standard.
- *            It can generate buggy code on targets which generate assembly depending on alignment.
+ *            It can generate buggy code on targets which assembly generation depends on alignment.
  *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
  * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
  * Prefer these methods in priority order (0 > 1 > 2)
  */
-#ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#ifndef LZ4_FORCE_MEMORY_ACCESS   /* can be defined externally */
 #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 #    define LZ4_FORCE_MEMORY_ACCESS 2
-#  elif defined(__INTEL_COMPILER) || \
-  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#  elif defined(__INTEL_COMPILER) || defined(__GNUC__)
 #    define LZ4_FORCE_MEMORY_ACCESS 1
 #  endif
 #endif
-- 
cgit v0.12


From e169edac054941c4019880d88b243821f8a10414 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 8 Apr 2017 18:27:31 -0700
Subject: lz4frame : control lz4 context creation success

---
 lib/lz4frame.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e04fe83..7c1b8fd 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -316,11 +316,11 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
     BYTE* dstPtr = dstStart;
     BYTE* const dstEnd = dstStart + dstCapacity;
 
-    memset(&cctxI, 0, sizeof(cctxI));   /* works because no allocation */
+    memset(&cctxI, 0, sizeof(cctxI));
     memset(&options, 0, sizeof(options));
 
     cctxI.version = LZ4F_VERSION;
-    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
+    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works because autoflush==1 & stableSrc==1 */
 
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
@@ -332,7 +332,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
     if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
         cctxI.lz4CtxPtr = &lz4ctx;
         cctxI.lz4CtxLevel = 1;
-    }
+    }  /* otherwise : will be created within LZ4F_compressBegin */
 
     prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
     prefs.autoFlush = 1;
@@ -341,7 +341,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
 
     options.stableSrc = 1;
 
-    if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))
+    if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))   /* condition to guarantee success */
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
 
     { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs);  /* write header */
@@ -356,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)   /* no allocation done with lz4 fast */
+    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)   /* Ctx allocation only for lz4hc */
         FREEMEM(cctxI.lz4CtxPtr);
 
     return (dstPtr - dstStart);
@@ -423,7 +423,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
 
-    /* ctx Management */
+    /* Ctx Management */
     {   U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
         if (cctxPtr->lz4CtxLevel < tableID) {
             FREEMEM(cctxPtr->lz4CtxPtr);
@@ -431,6 +431,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
             else
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
+            if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->lz4CtxLevel = tableID;
         }
     }
-- 
cgit v0.12


From 7eecd32c079cdf1afda8c00bba83dd3a1fee3f62 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 9 Apr 2017 01:11:39 -0700
Subject: ensure lz4f_cctx internal buffer size remain valid in case of malloc
 error

---
 lib/lz4frame.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7c1b8fd..fc0c7b5 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -445,10 +445,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
         requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB;   /* just needs dict */
 
     if (cctxPtr->maxBufferSize < requiredBuffSize) {
-        cctxPtr->maxBufferSize = requiredBuffSize;
+        cctxPtr->maxBufferSize = 0;
         FREEMEM(cctxPtr->tmpBuff);
         cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
         if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+        cctxPtr->maxBufferSize = requiredBuffSize;
     }
     cctxPtr->tmpIn = cctxPtr->tmpBuff;
     cctxPtr->tmpInSize = 0;
-- 
cgit v0.12


From e2c9b19122a19fa8a46a253e4ce556a5b75fa8b1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 9 Apr 2017 01:41:36 -0700
Subject: lz4frame : Added negative compression levels

---
 NEWS              |  1 +
 lib/lz4.h         | 14 +++++++-------
 lib/lz4frame.c    | 12 ++++++------
 lib/lz4frame.h    |  2 +-
 tests/frametest.c | 12 +++++++++++-
 5 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/NEWS b/NEWS
index 4fb8fdd..32fb613 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@ v1.7.6
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
 cli : added GNU separator -- specifying that all following arguments are files
 API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
+API : lz4frame : negative compression levels trigger fast acceleration
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
diff --git a/lib/lz4.h b/lib/lz4.h
index 588de22..d5a3016 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -256,19 +256,19 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
 
 /*! LZ4_compress_fast_continue() :
  *  Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- *  Important : Previous data blocks are assumed to still be present and unmodified !
+ *  Important : Previous data blocks are assumed to remain present and unmodified !
  *  'dst' buffer must be already allocated.
- *  If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
  *  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ *  After an error, the stream status is invalid, and it can only be reset or freed.
  */
-LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 /*! LZ4_saveDict() :
- *  If previously compressed data block is not guaranteed to remain available at its memory location,
+ *  If previously compressed data block is not guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
- *  Note : you don't need to call LZ4_loadDict() afterwards,
- *         dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
- *  Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+ *  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
+ *  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
  */
 LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index fc0c7b5..356be8a 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -517,14 +517,14 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com
 
 static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
 {
-    (void) level;
-    return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, 1);
+    int const acceleration = (level < -1) ? -level : 1;
+    return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
 {
-    (void) level;
-    return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, 1);
+    int const acceleration = (level < -1) ? -level : 1;
+    return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
 static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level)
@@ -619,7 +619,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
         if (compressOptionsPtr->stableSrc) {
             cctxPtr->tmpIn = cctxPtr->tmpBuff;
         } else {
-            int realDictSize = LZ4F_localSaveDict(cctxPtr);
+            int const realDictSize = LZ4F_localSaveDict(cctxPtr);
             if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
             cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
         }
@@ -629,7 +629,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)   /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
         && !(cctxPtr->prefs.autoFlush))
     {
-        int realDictSize = LZ4F_localSaveDict(cctxPtr);
+        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
         cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
     }
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 76b4e69..1b3e498 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -167,7 +167,7 @@ typedef struct {
  * All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 count as 0 */
+  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
   unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
diff --git a/tests/frametest.c b/tests/frametest.c
index ae8416b..a4c548e 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -218,6 +218,16 @@ int basicTests(U32 seed, double compressibility)
 
     /* test one-pass frame compression */
     testSize = COMPRESSIBLE_NOISE_LENGTH;
+
+    DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : ");
+    {   LZ4F_preferences_t fastCompressPrefs;
+        memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
+        fastCompressPrefs.compressionLevel = -3;
+        cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs);
+        if (LZ4F_isError(cSize)) goto _output_error;
+        DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
+    }
+
     DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
     if (LZ4F_isError(cSize)) goto _output_error;
@@ -618,7 +628,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
         prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
         prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
-        prefs.compressionLevel = FUZ_rand(&randState) % 5;
+        prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11);
         if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
 
         DISPLAYUPDATE(2, "\r%5u   ", testNb);
-- 
cgit v0.12


From 0b3e807043514ca4d232d9a92fd261fedb3afb13 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 26 Apr 2017 10:03:23 -0700
Subject: [LZ4F] Allow users to disable LZ4F_DEPRECATE

---
 lib/lz4frame.h | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 1b3e498..2e79a17 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -75,12 +75,16 @@ extern "C" {
 #  define LZ4FLIB_API
 #endif
 
-#if defined(_MSC_VER)
-#  define LZ4F_DEPRECATE(x) x   /* __declspec(deprecated) x - only works with C++ */
-#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
-#  define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
+#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS
+#  define LZ4F_DEPRECATE(x) x
 #else
-#  define LZ4F_DEPRECATE(x) x   /* no deprecation warning for this compiler */
+#  if defined(_MSC_VER)
+#    define LZ4F_DEPRECATE(x) x   /* __declspec(deprecated) x - only works with C++ */
+#  elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
+#    define LZ4F_DEPRECATE(x) x __attribute__((deprecated))
+#  else
+#    define LZ4F_DEPRECATE(x) x   /* no deprecation warning for this compiler */
+#  endif
 #endif
 
 
-- 
cgit v0.12


From f3460fc148fc828b9c33b1a3967b6e494ad1f8f2 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin 
Date: Sat, 29 Apr 2017 15:00:47 +0300
Subject: liz4hc.h: fix a comment: LZ4HC_MAX_CLEVEL -> LZ4HC_CLEVEL_MAX

---
 lib/lz4hc.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 2e3880d..c4284c0 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -58,8 +58,8 @@ extern "C" {
  * `dst` must be already allocated.
  * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
  * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
- * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work.
- *                      Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL.
+ * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work.
+ *                      Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
  */
-- 
cgit v0.12


From a41df535f709fcb6ccba079dd4d630c4ec4f7151 Mon Sep 17 00:00:00 2001
From: Alexey Tourbin 
Date: Sat, 29 Apr 2017 15:16:51 +0300
Subject: lz4cli.c: fix a comment: LZ4HC_DEFAULT_CLEVEL -> LZ4HC_CLEVEL_MAX

Actually the program only mentions LZ4HC_CLEVEL_MAX.
---
 programs/lz4cli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 329ca0b..10aa5aa 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -49,7 +49,7 @@
 #include    /* strcmp, strlen */
 #include "bench.h"    /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
 #include "lz4io.h"    /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */
-#include "lz4hc.h"    /* LZ4HC_DEFAULT_CLEVEL */
+#include "lz4hc.h"    /* LZ4HC_CLEVEL_MAX */
 #include "lz4.h"      /* LZ4_VERSION_STRING */
 
 
-- 
cgit v0.12


From 6cf1f7d758c23d6033d388eabbbb50380c364d2e Mon Sep 17 00:00:00 2001
From: Alexey Tourbin 
Date: Sat, 29 Apr 2017 15:52:51 +0300
Subject: lz4hc.c: clamp compression levels > 12

I noticed that, while 'lz4 -12' works fine, 'lz4 -13' does not compress
at all.

$ cat searchNum = LZ4HC_getSearchNum(cLevel);
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
             default:
+                cLevel = 12;
+                /* pass-through */
             case 12:
                 ctx->searchNum = LZ4HC_getSearchNum(cLevel);
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1);
-- 
cgit v0.12


From a8dd86d93e963edd6761936377dff21b45016156 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 1 May 2017 22:32:21 -0700
Subject: changed macro HEAPMODE into LZ4_HEAPMODE

This macro is susceptible to be triggered from user side
typically through compiler flag (-DLZ4_HEAPMODE=1).
In which case, it makes sense to prefix the macro
since we want to reduce potential side-effect on namespace.
---
 lib/lz4.c    | 14 +++++++-------
 lib/lz4opt.h | 13 +++++++------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index c9c5a07..93de1e5 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -37,12 +37,12 @@
 *  Tuning parameters
 **************************************/
 /*
- * HEAPMODE :
+ * LZ4_HEAPMODE :
  * Select how default compression functions will allocate memory for their hash table,
  * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
  */
-#ifndef HEAPMODE
-#  define HEAPMODE 0
+#ifndef LZ4_HEAPMODE
+#  define LZ4_HEAPMODE 0
 #endif
 
 /*
@@ -677,7 +677,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int
 
 int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
     void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctx;
@@ -686,7 +686,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp
 
     int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
 
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
     FREEMEM(ctxPtr);
 #endif
     return result;
@@ -890,7 +890,7 @@ static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src,
 
 int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
     LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t));   /* malloc-calloc always properly aligned */
 #else
     LZ4_stream_t ctxBody;
@@ -899,7 +899,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
 
     int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
 
-#if (HEAPMODE)
+#if (LZ4_HEAPMODE)
     FREEMEM(ctx);
 #endif
     return result;
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 416241a..b346eba 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -49,11 +49,12 @@ typedef struct {
 } LZ4HC_optimal_t;
 
 
-/* price in bits */
+/* price in bytes */
 FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 {
     size_t price = litlen;
-    if (litlen >= (size_t)RUN_MASK) price += 1 + (litlen-RUN_MASK)/255;
+    if (litlen >= (size_t)RUN_MASK)
+        price += 1 + (litlen-RUN_MASK)/255;
     return price;
 }
 
@@ -66,7 +67,7 @@ FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
     price += LZ4HC_literalsPrice(litlen);
 
     if (mlen >= (size_t)(ML_MASK+MINMATCH))
-        price+= 1+(mlen-(ML_MASK+MINMATCH))/255;
+        price+= 1 + (mlen-(ML_MASK+MINMATCH))/255;
 
     return price;
 }
@@ -232,7 +233,7 @@ static int LZ4HC_compress_optimal (
         size_t const llen = ip - anchor;
         size_t last_pos = 0;
         size_t match_num, cur, best_mlen, best_off;
-        memset(opt, 0, sizeof(LZ4HC_optimal_t));
+        memset(opt, 0, sizeof(LZ4HC_optimal_t));  /* memset only the first one */
 
         match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!match_num) { ip++; continue; }
@@ -279,7 +280,7 @@ static int LZ4HC_compress_optimal (
                 }
 
                 if (price < (size_t)opt[cur].price)
-                    SET_PRICE(cur, 1, 0, litlen, price);   /* note : increases last_pos */
+                    SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price);   /* note : increases last_pos */
             }
 
             if (cur == last_pos || curPtr >= mflimit) break;
@@ -331,7 +332,7 @@ encode: /* cur, last_pos, best_mlen, best_off must be set */
             opt[cur].off = (int)best_off;
             best_mlen = ml;
             best_off = offset;
-            if (ml > cur) break;
+            if (ml > cur) break;   /* can this happen ? */
             cur -= ml;
         }
 
-- 
cgit v0.12


From 1efa48831e4e2422711262e5d3c3cbf613189d3b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 2 May 2017 12:01:13 -0700
Subject: minor readability changes

---
 lib/lz4hc.c | 42 ++++++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 16fe029..b987d39 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -69,9 +69,9 @@
 
 
 /*===   Macros   ===*/
-#define HASH_FUNCTION(i)       (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
-#define DELTANEXTMAXD(p)       chainTable[(p) & LZ4HC_MAXD_MASK]    /* flexible, LZ4HC_MAXD dependent */
-#define DELTANEXTU16(p)        chainTable[(U16)(p)]   /* faster */
+#define HASH_FUNCTION(i)         (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
+#define DELTANEXTMAXD(p)         chainTable[(p) & LZ4HC_MAXD_MASK]    /* flexible, LZ4HC_MAXD dependent */
+#define DELTANEXTU16(table, pos) table[(U16)(pos)]   /* faster */
 
 static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
 
@@ -106,7 +106,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
         U32 const h = LZ4HC_hashPtr(base+idx);
         size_t delta = idx - hashTable[h];
         if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
-        DELTANEXTU16(idx) = (U16)delta;
+        DELTANEXTU16(chainTable, idx) = (U16)delta;
         hashTable[h] = idx;
         idx++;
     }
@@ -115,8 +115,8 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
 }
 
 
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4,   /* Index table will be updated */
-                                               const BYTE* ip, const BYTE* const iLimit,
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
+                                               const BYTE* const ip, const BYTE* const iLimit,
                                                const BYTE** matchpos,
                                                const int maxNbAttempts)
 {
@@ -138,8 +138,8 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4,   /* In
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const match = base + matchIndex;
-            if (*(match+ml) == *(ip+ml)
-                && (LZ4_read32(match) == LZ4_read32(ip)))
+            if ( (*(match+ml) == *(ip+ml))   /* can be longer */
+               && (LZ4_read32(match) == LZ4_read32(ip)) )
             {
                 size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
                 if (mlt > ml) { ml = mlt; *matchpos = match; }
@@ -156,7 +156,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4,   /* In
                 if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
             }
         }
-        matchIndex -= DELTANEXTU16(matchIndex);
+        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
     }
 
     return (int)ml;
@@ -180,9 +180,9 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const lowPrefixPtr = base + dictLimit;
     const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
     const BYTE* const dictBase = hc4->dictBase;
-    U32   matchIndex;
+    int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
-    int delta = (int)(ip-iLowLimit);
+    U32   matchIndex;
 
 
     /* First Match */
@@ -206,14 +206,14 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                     mlt -= back;
 
                     if (mlt > longest) {
-                        longest = (int)mlt;
+                        longest = mlt;
                         *matchpos = matchPtr+back;
                         *startpos = ip+back;
             }   }   }
         } else {
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
-                size_t mlt;
+                int mlt;
                 int back=0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
@@ -222,10 +222,10 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                     mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
                 while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
                 mlt -= back;
-                if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
+                if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
             }
         }
-        matchIndex -= DELTANEXTU16(matchIndex);
+        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
     }
 
     return longest;
@@ -238,12 +238,10 @@ typedef enum {
     limitedDestSize = 2,
 } limitedOutput_directive;
 
-#define LZ4HC_DEBUG 0
-#if LZ4HC_DEBUG
-static unsigned debug = 0;
+#ifndef LZ4HC_DEBUG
+#  define LZ4HC_DEBUG 0
 #endif
 
-
 /* LZ4HC_encodeSequence() :
  * @return : 0 if ok,
  *           1 if buffer issue detected */
@@ -257,15 +255,15 @@ FORCE_INLINE int LZ4HC_encodeSequence (
     BYTE* oend)
 {
     size_t length;
-    BYTE* token;
+    BYTE* const token = (*op)++;
 
 #if LZ4HC_DEBUG
-    if (debug) printf("literal : %u  --  match : %u  --  offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
+    printf("literal : %u  --  match : %u  --  offset : %u\n",
+           (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
 #endif
 
     /* Encode Literal length */
     length = (size_t)(*ip - *anchor);
-    token = (*op)++;
     if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1;   /* Check output limit */
     if (length >= RUN_MASK) {
         size_t len = length - RUN_MASK;
-- 
cgit v0.12


From fe932c4527aabd1c642b67fddf90e9be8ea380f8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 12:25:05 -0700
Subject: expose LZ4F_resetDecompressionContext()

---
 lib/lz4frame.c | 134 +++++++++++++++++++++++++++++----------------------------
 lib/lz4frame.h |  11 +++++
 2 files changed, 80 insertions(+), 65 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 356be8a..7422638 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -209,7 +209,8 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
 
 static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
 {
-    LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));    /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+    /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
+    LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
     return (LZ4F_errorCode_t)-(ptrdiff_t)code;
 }
 
@@ -241,7 +242,8 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length)
 /*-************************************
 *  Simple-pass compression functions
 **************************************/
-static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize)
+static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
+                                           const size_t srcSize)
 {
     LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
     size_t maxBlockSize = 64 KB;
@@ -256,11 +258,13 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
 
 /* LZ4F_compressBound() :
  * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
- * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
+ * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
+ * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
  * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  */
-static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered)
+static size_t LZ4F_compressBound_internal(size_t srcSize,
+                                    const LZ4F_preferences_t* preferencesPtr,
+                                          size_t alreadyBuffered)
 {
     LZ4F_preferences_t prefsNull;
     memset(&prefsNull, 0, sizeof(prefsNull));
@@ -298,14 +302,14 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
 
 
 /*! LZ4F_compressFrame() :
-* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
-* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
-* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
-* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
-* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default.
-* The result of the function is the number of bytes written into dstBuffer.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
+ *  Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
+ *  The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
+ *  If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
+ *  Get the minimum value of dstCapacity by using LZ4F_compressFrameBound().
+ *  The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default.
+ *  The result of the function is the number of bytes written into dstBuffer.
+ *  The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ */
 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
     LZ4F_cctx_t cctxI;
@@ -404,10 +408,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 
 
 /*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
- * @return : number of bytes written into dstBuffer for the header
- *           or an error code (can be tested using LZ4F_isError())
+ *  will write the frame header into dstBuffer.
+ *  dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+ *  @return : number of bytes written into dstBuffer for the header
+ *            or an error code (can be tested using LZ4F_isError())
  */
 size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
 {
@@ -650,13 +654,13 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
 
 
 /*! LZ4F_flush() :
-* Should you need to create compressed data immediately, without waiting for a block to be filled,
-* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
-* The result of the function is the number of bytes written into dstBuffer
-* (it can be zero, this means there was no data left within compressionContext)
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-*/
+ *  Should you need to create compressed data immediately, without waiting for a block to be filled,
+ *  you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
+ *  The result of the function is the number of bytes written into dstBuffer
+ *  (it can be zero, this means there was no data left within compressionContext)
+ *  The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ *  The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ */
 size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
 {
     BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -687,14 +691,14 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
 
 
 /*! LZ4F_compressEnd() :
-* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
-* It will flush whatever data remained within compressionContext (like LZ4_flush())
-* but also properly finalize the frame, with an endMark and a checksum.
-* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
-*/
+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ4_flush())
+ * but also properly finalize the frame, with an endMark and a checksum.
+ * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
+ */
 size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
 {
     BYTE* const dstStart = (BYTE*)dstBuffer;
@@ -751,11 +755,11 @@ struct LZ4F_dctx_s {
 
 
 /*! LZ4F_createDecompressionContext() :
-*   Create a decompressionContext object, which will track all decompression operations.
-*   Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
-*   Object can later be released using LZ4F_freeDecompressionContext().
-*   @return : if != 0, there was an error during context creation.
-*/
+ *   Create a decompressionContext object, which will track all decompression operations.
+ *   Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
+ *   Object can later be released using LZ4F_freeDecompressionContext().
+ *   @return : if != 0, there was an error during context creation.
+ */
 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
     LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
@@ -802,9 +806,9 @@ LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
 
 
 /*! LZ4F_headerSize() :
-*   @return : size of frame header
-*             or an error code, which can be tested using LZ4F_isError()
-*/
+ *   @return : size of frame header
+ *             or an error code, which can be tested using LZ4F_isError()
+ */
 static size_t LZ4F_headerSize(const void* src, size_t srcSize)
 {
     /* minimal srcSize to determine header size */
@@ -825,13 +829,13 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
 
 
 /*! LZ4F_decodeHeader() :
-   input   : `src` points at the **beginning of the frame**
-   output  : set internal values of dctx, such as
-             dctxPtr->frameInfo and dctxPtr->dStage.
-             Also allocates internal buffers.
-   @return : nb Bytes read from src (necessarily <= srcSize)
-             or an error code (testable with LZ4F_isError())
-*/
+ *  input   : `src` points at the **beginning of the frame**
+ *  output  : set internal values of dctx, such as
+ *            dctxPtr->frameInfo and dctxPtr->dStage.
+ *            Also allocates internal buffers.
+ *  @return : nb Bytes read from src (necessarily <= srcSize)
+ *            or an error code (testable with LZ4F_isError())
+ */
 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
 {
     unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
@@ -913,8 +917,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
 
 
 /*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. The objective is to provide relevant information for allocation purposes.
+ * This function extracts frame parameters (max blockSize, frame checksum, etc.).
+ * Usage is optional. Objective is to provide relevant information for allocation purposes.
  * This function works in 2 situations :
  *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
  *     Amount of input data provided must be large enough to successfully decode the frame header.
@@ -1028,22 +1032,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi
 
 
 /*! LZ4F_decompress() :
-* Call this function repetitively to regenerate data compressed within srcBuffer.
-* The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
-*
-* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
-*
-* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
-* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
-* Remaining data will have to be presented again in a subsequent invocation.
-*
-* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
-* Basically, it's the size of the current (or remaining) compressed block + header of next block.
-* Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
-* Note that this is just a hint, it's always possible to any srcSize value.
-* When a frame is fully decoded, @return will be 0.
-* If decompression failed, @return is an error code which can be tested using LZ4F_isError().
-*/
+ * Call this function repetitively to regenerate data compressed within srcBuffer.
+ * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *
+ * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
+ * Remaining data will have to be presented again in a subsequent invocation.
+ *
+ * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
+ * Basically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
+ * Note that this is just a hint, it's always possible to any srcSize value.
+ * When a frame is fully decoded, @return will be 0.
+ * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
+ */
 size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                        void* dstBuffer, size_t* dstSizePtr,
                        const void* srcBuffer, size_t* srcSizePtr,
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 2e79a17..b61fe7f 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -363,6 +363,17 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    const LZ4F_decompressOptions_t* dOptPtr);
 
 
+/*! LZ4F_resetDecompressionContext() :
+ *  When decompression ends successfully,
+ *  it's possible to start a new decompression immediately
+ *  re-using the same context.
+ *  However, in case of an error, the context is left in "undefined" state.
+ *  In which case, it's necessary to reset it, before re-using it.
+ *  This method can also be used to abruptly stop an unfinished decompression,
+ *  and start a new on the same context. */
+LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+
+
 
 #if defined (__cplusplus)
 }
-- 
cgit v0.12


From b8575f2d2b1e05c20812215759b29bf2d0f7a708 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 13:26:04 -0700
Subject: updated Makefile

to automatically build manual files with
make all
---
 Makefile                      | 81 ++++++++++++++++++++-----------------------
 contrib/gen_manual/.gitignore |  2 ++
 contrib/gen_manual/Makefile   | 26 +++++++++++---
 doc/lz4_manual.html           | 18 +++++-----
 doc/lz4frame_manual.html      | 75 +++++++++++++++++++++++++++------------
 lib/Makefile                  | 13 +++----
 lib/lz4frame.h                |  2 +-
 programs/Makefile             | 26 ++++++--------
 8 files changed, 140 insertions(+), 103 deletions(-)
 create mode 100644 contrib/gen_manual/.gitignore

diff --git a/Makefile b/Makefile
index 1f5165d..24328d5 100644
--- a/Makefile
+++ b/Makefile
@@ -52,36 +52,36 @@ EXT =
 endif
 
 
-.PHONY: default all lib lz4 clean test versionsTest examples
+.PHONY: default
+default: lib lz4-release
 
-default:
-	@$(MAKE) -C $(LZ4DIR)
-	@$(MAKE) -C $(PRGDIR)
-	@cp $(PRGDIR)/lz4$(EXT) .
-
-all:
-	@$(MAKE) -C $(LZ4DIR) $@
-	@$(MAKE) -C $(PRGDIR) $@
-	@$(MAKE) -C $(TESTDIR) $@
-	@$(MAKE) -C $(EXDIR) $@
+.PHONY: all
+all: default examples manuals
 
+.PHONY: lib
 lib:
 	@$(MAKE) -C $(LZ4DIR)
 
-lz4:
+.PHONY: lz4 lz4-release
+lz4 lz4-release: lib
 	@$(MAKE) -C $(PRGDIR) $@
 	@cp $(PRGDIR)/lz4$(EXT) .
 
-lz4-release:
-	@$(MAKE) -C $(PRGDIR)
-	@cp $(PRGDIR)/lz4$(EXT) .
+.PHONY: examples
+examples: lib lz4-release
+	$(MAKE) -C $(EXDIR) test
 
+.PHONY: manuals
+manuals:
+	@$(MAKE) -C contrib/gen_manual $@
+
+.PHONY: clean
 clean:
+	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
-	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
 	@$(MAKE) -C $(EXDIR) $@ > $(VOID)
-	@$(MAKE) -C examples $@ > $(VOID)
+	@$(MAKE) -C contrib/gen_manual $@
 	@$(RM) lz4$(EXT)
 	@echo Cleaning completed
 
@@ -92,17 +92,31 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
 
-install:
-	@$(MAKE) -C $(LZ4DIR) $@
-	@$(MAKE) -C $(PRGDIR) $@
-
-uninstall:
+install uninstall:
 	@$(MAKE) -C $(LZ4DIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
 
 travis-install:
 	$(MAKE) -j1 install PREFIX=~/install_test_dir
 
+cmake:
+	@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
+
+endif
+
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+CMAKE_PARAMS = -G"MSYS Makefiles"
+endif
+
+
+#------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+
+.PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
 
@@ -135,31 +149,10 @@ platformTest: clean
 	CFLAGS="-O3 -Werror -static" $(MAKE) -C $(TESTDIR) all
 	$(MAKE) -C $(TESTDIR) test-platform
 
+.PHONY: versionsTest
 versionsTest: clean
 	$(MAKE) -C $(TESTDIR) $@
 
-examples:
-	$(MAKE) -C $(LZ4DIR)
-	$(MAKE) -C $(PRGDIR) lz4
-	$(MAKE) -C examples test
-
-endif
-
-
-ifneq (,$(filter MSYS%,$(shell uname)))
-HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles"
-endif
-
-
-#------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
-#------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-
-cmake:
-	@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
-
 gpptest: clean
 	g++ -v
 	CC=g++ $(MAKE) -C $(LZ4DIR)  all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
diff --git a/contrib/gen_manual/.gitignore b/contrib/gen_manual/.gitignore
new file mode 100644
index 0000000..6ea967f
--- /dev/null
+++ b/contrib/gen_manual/.gitignore
@@ -0,0 +1,2 @@
+# build artefact
+gen_manual
diff --git a/contrib/gen_manual/Makefile b/contrib/gen_manual/Makefile
index adbcca2..9fbe858 100644
--- a/contrib/gen_manual/Makefile
+++ b/contrib/gen_manual/Makefile
@@ -35,7 +35,15 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -W
 CFLAGS += $(MOREFLAGS)
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
-
+LZ4API = ../../lib/lz4.h
+LZ4MANUAL = ../../doc/lz4_manual.html
+LZ4FAPI = ../../lib/lz4frame.h
+LZ4FMANUAL = ../../doc/lz4frame_manual.html
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LZ4VER := $(shell echo $(LIBVER_SCRIPT))
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -45,14 +53,24 @@ EXT =
 endif
 
 
-.PHONY: default gen_manual
-
+.PHONY: default
 default: gen_manual
 
 gen_manual: gen_manual.cpp
-	$(CXX)      $(FLAGS) $^ -o $@$(EXT)
+	$(CXX) $(FLAGS) $^ -o $@$(EXT)
+
+$(LZ4MANUAL) : gen_manual $(LZ4API)
+	echo "Update lz4 manual in /doc"
+	./gen_manual $(LZ4VER) $(LZ4API) $@
+
+$(LZ4FMANUAL) : gen_manual $(LZ4FAPI)
+	echo "Update lz4frame manual in /doc"
+	./gen_manual $(LZ4VER) $(LZ4FAPI) $@
 
+.PHONY: manuals
+manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
 
+.PHONY: clean
 clean:
 	@$(RM) gen_manual$(EXT)
 	@echo Cleaning completed
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ff6e149..4c4174d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 
 
 
-lz4 1.7.5 Manual
+1.7.6 Manual
 
 
-
lz4 1.7.5 Manual

+
1.7.6 Manual

 

 
Contents

 
    
@@ -170,21 +170,21 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
  
 


 
-
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
  Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
-  Important : Previous data blocks are assumed to still be present and unmodified !
+  Important : Previous data blocks are assumed to remain present and unmodified !
   'dst' buffer must be already allocated.
-  If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
   If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+  After an error, the stream status is invalid, and it can only be reset or freed.
  
 


 
 
int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
-
  If previously compressed data block is not guaranteed to remain available at its memory location,
+
  If previously compressed data block is not guaranteed to remain available at its current memory location,
   save it into a safer place (char* safeBuffer).
-  Note : you don't need to call LZ4_loadDict() afterwards,
-         dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue().
-  Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
+  Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable.
+  @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
  
 


 
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index f76ec3d..5d553ed 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,19 +1,23 @@
 
 
 
-lz4frame 1.7.5 Manual
+1.7.6 Manual
 
 
-
lz4frame 1.7.5 Manual

+
1.7.6 Manual

 

 
Contents

 
    
 
  1. Introduction
    2. 
-
  2. Error management
    3. 
-
  3. Frame compression types
    4. 
-
  4. Simple compression function
    5. 
-
  5. Advanced compression functions
    6. 
-
  6. Decompression functions
    7. 
+
  7. Compiler specifics
    8. 
+
  8. Error management
    9. 
+
  9. Frame compression types
    10. 
+
  10. Simple compression function
    11. 
+
  11. Advanced compression functions
    12. 
+
  12. Resource Management
    13. 
+
  13. Compression
    14. 
+
  14. Decompression functions
    15. 
+
  15. Streaming decompression functions
    16. 
 

 

 
Introduction
@@ -22,13 +26,15 @@
   of encoding standard metadata alongside LZ4-compressed blocks.
 

 
-
Error management

+
Compiler specifics

+
+
Error management

 
 
unsigned    LZ4F_isError(LZ4F_errorCode_t code);   /**< tells if a `LZ4F_errorCode_t` function result is an error code */
 


 
const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return error code string; useful for debugging */
 


-
Frame compression types

+
Frame compression types

 
 
typedef enum {
     LZ4F_default=0,
@@ -77,7 +83,7 @@
 
 
typedef struct {
   LZ4F_frameInfo_t frameInfo;
-  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+  int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
   unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
@@ -86,7 +92,7 @@
  All reserved fields must be set to zero. 
 


 
-
Simple compression function

+
Simple compression function

 
 
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
 Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
@@ -105,17 +111,19 @@
  
 


 
-
Advanced compression functions

+
Advanced compression functions

 
 
typedef struct {
   unsigned stableSrc;    /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */
   unsigned reserved[3];
 } LZ4F_compressOptions_t;
 


+
Resource Management

+
 
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version);
 LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.
  The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL.
  The function will provide a pointer to a fully allocated LZ4F_cctx object.
  If @return != zero, there was an error during context creation.
@@ -123,6 +131,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  
 


 
+
Compression

+
 
size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
 
 will write the frame header into dstBuffer.
  dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
@@ -173,7 +183,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  
 


 
-
Decompression functions

+
Decompression functions

 
 
typedef struct {
   unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
@@ -181,7 +191,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 } LZ4F_decompressOptions_t;
 


 
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
 Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
  The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
  The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
@@ -192,19 +202,27 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
  
 


 
+
Streaming decompression functions

+
 
size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
-
 This function decodes frame header information (such as max blockSize, frame checksum, etc.).
- Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
- A header size is variable and can length from 7 to 15 bytes. It's possible to provide more input bytes than that.
+
 This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
+ Its usage is optional. Extracted information can be useful for allocation purposes, typically.
+ This function works in 2 situations :
+   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+     Input size must be large enough to successfully decode the entire frame header.
+     Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
+     It's allowed to provide more input data than this minimum.
+   - After decoding has been started.
+     In which case, no input is read, frame parameters are extracted from dctx.
+     If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- Decompression must resume from this point (srcBuffer + *srcSizePtr).
- Note that LZ4F_getFrameInfo() can also be used anytime *after* decompression is started, in which case 0 input byte can be enough.
- Frame header info is *copied into* an already allocated LZ4F_frameInfo_t structure.
+ Decompression must resume from (srcBuffer + *srcSizePtr).
  @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
            or an error code which can be tested using LZ4F_isError()
-           (typically, when there is not enough src bytes to fully decode the frame header)
+ note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  
 


 
@@ -227,14 +245,25 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* const dctx);
 
  @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
  This is just a hint though, it's always possible to provide any srcSize.
  When a frame is fully decoded, @return will be 0 (no more data expected).
  If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
 
  After a frame is fully decoded, dctx can be used again to decompress another frame.
+ After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  
 


 
+
LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+
  When decompression ends successfully,
+  it's possible to start a new decompression immediately
+  re-using the same context.
+  However, in case of an error, the context is left in "undefined" state.
+  In which case, it's necessary to reset it, before re-using it.
+  This method can also be used to abruptly stop an unfinished decompression,
+  and start a new on the same context. 
+


+
 
 
diff --git a/lib/Makefile b/lib/Makefile
index 9a794b8..c6fd7b8 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -46,10 +46,10 @@ BUILD_STATIC:= yes
 
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
 CFLAGS  ?= -O3
-DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
            -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS  += $(MOREFLAGS)
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
@@ -70,12 +70,13 @@ endif
 
 LIBLZ4 = liblz4.$(SHARED_EXT_VER)
 
+.PHONY: default
 default: lib-release
 
-lib-release: liblz4.a liblz4
+lib-release: DEBUGFLAGS :=
+lib-release: lib
 
-lib: CFLAGS += $(DEBUGFLAGS)
-lib: lib-release
+lib: liblz4.a liblz4
 
 all: lib
 
@@ -83,7 +84,7 @@ all32: CFLAGS+=-m32
 all32: all
 
 liblz4.a: *.c
-ifeq ($(BUILD_STATIC),yes)
+ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 	@echo compiling static library
 	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
 	@$(AR) rcs $@ *.o
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index b61fe7f..9a2130a 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -363,7 +363,7 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    const LZ4F_decompressOptions_t* dOptPtr);
 
 
-/*! LZ4F_resetDecompressionContext() :
+/*! LZ4F_resetDecompressionContext() : v1.8.0
  *  When decompression ends successfully,
  *  it's possible to start a new decompression immediately
  *  re-using the same context.
diff --git a/programs/Makefile b/programs/Makefile
index 4a8103c..2e73780 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -51,7 +51,7 @@ CFLAGS  ?= -O3
 DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS  += $(MOREFLAGS)
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 LZ4_VERSION=$(LIBVER)
@@ -75,28 +75,17 @@ all: lz4 lz4c
 all32: CFLAGS+=-m32
 all32: all
 
-lz4: CFLAGS += $(DEBUGFLAGS)
 lz4: $(OBJFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 lz4-release: DEBUGFLAGS=
 lz4-release: lz4
 
-lz4c  : CFLAGS += $(DEBUGFLAGS)
-lz4c  : $(SRCFILES)
-	$(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
-
-lz4c32: CFLAGS += -m32 $(DEBUGFLAGS)
-lz4c32: $(SRCFILES)
+lz4c32: CFLAGS += -m32
+lz4c: CPPFLAGS += -DENABLE_LZ4C_LEGACY_OPTIONS
+lz4c lz4c32 : $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-clean:
-	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
-	@$(RM) core *.o *.test tmp* \
-           lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
-	@echo Cleaning completed
-
-
 lz4.1: lz4.1.md
 	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 
@@ -108,6 +97,12 @@ clean-man:
 preview-man: clean-man man
 	man ./lz4.1
 
+clean:
+	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
+	@$(RM) core *.o *.test tmp* \
+           lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat
+	@echo Cleaning completed
+
 
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
@@ -120,7 +115,6 @@ unlz4: lz4
 lz4cat: lz4
 	ln -s lz4 lz4cat
 
-
 ifneq (,$(filter $(shell uname),SunOS))
 INSTALL ?= ginstall
 else
-- 
cgit v0.12


From a902127613921408bb02b1debe9371e9bcfaf9f2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 13:27:18 -0700
Subject: bumped version number to 1.8.0

due to addition of prototype
LZ4F_resetDecompressionContext()
---
 doc/lz4_manual.html      | 4 ++--
 doc/lz4frame_manual.html | 4 ++--
 lib/lz4.h                | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 4c4174d..fecd8cd 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.7.6 Manual
+1.8.0 Manual
 
 
-
1.7.6 Manual

+
1.8.0 Manual

 

 
Contents

 
    
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 5d553ed..fa2c0b9 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.7.6 Manual
+1.8.0 Manual
 
 
-
    1.7.6 Manual
    
+
    1.8.0 Manual
    
 
    
 
    Contents
    
 
      
diff --git a/lib/lz4.h b/lib/lz4.h
index d5a3016..5b6bc92 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -88,8 +88,8 @@ extern "C" {
 
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
-#define LZ4_VERSION_MINOR    7    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_MINOR    8    /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 
-- 
cgit v0.12


From 2012e4de9e2c33a5abb34d40082afc8918852c6e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 14:51:09 -0700
Subject: fixed c_standards tests

and added entry "make list"
---
 Makefile          | 24 ++++++++++++++----------
 tests/frametest.c |  3 ++-
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 24328d5..b31ebd0 100644
--- a/Makefile
+++ b/Makefile
@@ -56,7 +56,10 @@ endif
 default: lib lz4-release
 
 .PHONY: all
-all: default examples manuals
+all: allmost manuals
+
+.PHONY: allmost
+allmost: lib lz4 examples
 
 .PHONY: lib
 lib:
@@ -68,7 +71,7 @@ lz4 lz4-release: lib
 	@cp $(PRGDIR)/lz4$(EXT) .
 
 .PHONY: examples
-examples: lib lz4-release
+examples: lib lz4
 	$(MAKE) -C $(EXDIR) test
 
 .PHONY: manuals
@@ -116,6 +119,10 @@ endif
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 
+.PHONY: list
+list:
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
@@ -166,13 +173,10 @@ gpptest32: clean
 	CC=g++ $(MAKE) -C $(TESTDIR) native CFLAGS="-m32 -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
 c_standards: clean
-	$(MAKE) all MOREFLAGS="-std=gnu90 -Werror"
-	$(MAKE) clean
-	$(MAKE) all MOREFLAGS="-std=c99 -Werror"
-	$(MAKE) clean
-	$(MAKE) all MOREFLAGS="-std=gnu99 -Werror"
-	$(MAKE) clean
-	$(MAKE) all MOREFLAGS="-std=c11 -Werror"
-	$(MAKE) clean
+	# note : lz4 is not C90 compatible, because it requires long long support
+	CFLAGS="-std=gnu90 -Werror" $(MAKE) clean allmost
+	CFLAGS="-std=c99   -Werror" $(MAKE) clean allmost
+	CFLAGS="-std=gnu99 -Werror" $(MAKE) clean allmost
+	CFLAGS="-std=c11   -Werror" $(MAKE) clean allmost
 
 endif
diff --git a/tests/frametest.c b/tests/frametest.c
index a4c548e..04597a4 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -290,7 +290,8 @@ int basicTests(U32 seed, double compressibility)
             {   size_t nullSize = 0;
                 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
                 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
-                    DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
+                    DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
+                                    LZ4F_getErrorName(fiError));
                     goto _output_error;
                 }
                 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
-- 
cgit v0.12


From e60cbb5cacaa43eda7a3df7c2d9051a9437e7046 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 16:28:36 -0700
Subject: added test for LZ4F_resetDecompressionContext()

---
 doc/lz4frame_manual.html |  9 +++----
 lib/lz4frame.c           | 66 ++++++++++++++++++++++++++++++++----------------
 lib/lz4frame.h           |  9 +++----
 lib/lz4frame_static.h    |  9 -------
 tests/frametest.c        | 45 +++++++++++++++++++++------------
 5 files changed, 79 insertions(+), 59 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index fa2c0b9..87750a1 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -255,14 +255,11 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
  
 


 
-
LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
-
  When decompression ends successfully,
-  it's possible to start a new decompression immediately
-  re-using the same context.
-  However, in case of an error, the context is left in "undefined" state.
+
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
+
  In case of an error, the context is left in "undefined" state.
   In which case, it's necessary to reset it, before re-using it.
   This method can also be used to abruptly stop an unfinished decompression,
-  and start a new on the same context. 
+  and start a new with the same context. 
 


 
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 7422638..fb37789 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -798,10 +798,9 @@ typedef enum {
     dstage_skipSkippable
 } dStage_t;
 
-LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
+void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
 {
     dctx->dStage = dstage_getHeader;
-    return 0;
 }
 
 
@@ -939,7 +938,8 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
         size_t o=0, i=0;
         *srcSizePtr = 0;
         *frameInfoPtr = dctxPtr->frameInfo;
-        return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);  /* returns : recommended nb of bytes for LZ4F_decompress() */
+        /* returns : recommended nb of bytes for LZ4F_decompress() */
+        return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);
     } else {
         if (dctxPtr->dStage == dstage_storeHeader) {
             /* frame decoding already started, in the middle of header => automatic fail */
@@ -949,7 +949,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
             size_t decodeResult;
             size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
             if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
-            if (*srcSizePtr < hSize) { *srcSizePtr=0; return err0r(LZ4F_ERROR_frameHeader_incomplete); }
+            if (*srcSizePtr < hSize) {
+                *srcSizePtr=0;
+                return err0r(LZ4F_ERROR_frameHeader_incomplete);
+            }
 
             decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
             if (LZ4F_isError(decodeResult)) {
@@ -965,10 +968,15 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
 
 
 /* trivial redirector, for common prototype */
-static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
+static int LZ4F_decompress_safe (const char* src,
+                                 char* dst,
+                                 int compressedSize,
+                                 int dstCapacity,
+                                 const char* dictStart,
+                                 int dictSize)
 {
     (void)dictStart; (void)dictSize;
-    return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
+    return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity);
 }
 
 
@@ -1101,7 +1109,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                     doAnotherStage = 0;   /* not enough src data, ask for some more */
                     break;
                 }
-                {   LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
+                {   LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);  /* will change dStage appropriately */
                     if (LZ4F_isError(hSize)) return hSize;
                 }
                 break;
@@ -1149,7 +1157,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
                 dctxPtr->tmpInSize += sizeToCopy;
-                if (dctxPtr->tmpInSize < BHSize) {   /* not enough input to get full cBlockSize; wait for more */
+                if (dctxPtr->tmpInSize < BHSize) {   /* not enough input for cBlockSize */
                     nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
                     doAnotherStage  = 0;
                     break;
@@ -1157,13 +1165,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 selectedIn = dctxPtr->tmpIn;
             }
 
-        /* case dstage_decodeCBlockSize: */   /* no more direct access, to prevent scan-build warning */
+        /* case dstage_decodeCBlockSize: */   /* no more direct access, to remove scan-build warning */
             {   size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
                 if (nextCBlockSize==0) {  /* frameEnd signal, no more CBlock */
                     dctxPtr->dStage = dstage_getSuffix;
                     break;
                 }
-                if (nextCBlockSize > dctxPtr->maxBlockSize) return err0r(LZ4F_ERROR_GENERIC);   /* invalid cBlockSize */
+                if (nextCBlockSize > dctxPtr->maxBlockSize)
+                    return err0r(LZ4F_ERROR_maxBlockSize_invalid);
                 dctxPtr->tmpInTarget = nextCBlockSize;
                 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
                     dctxPtr->dStage = dstage_copyDirect;
@@ -1179,11 +1188,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
 
         case dstage_copyDirect:   /* uncompressed block */
             {   size_t sizeToCopy = dctxPtr->tmpInTarget;
-                if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;  /* not enough input to read full block */
+                if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;
                 if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
                 memcpy(dstPtr, srcPtr, sizeToCopy);
-                if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
-                if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy;
+                if (dctxPtr->frameInfo.contentChecksumFlag)
+                    XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
+                if (dctxPtr->frameInfo.contentSize)
+                    dctxPtr->frameRemainingSize -= sizeToCopy;
 
                 /* dictionary management */
                 if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@@ -1244,10 +1255,14 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 else
                     decoder = LZ4F_decompress_safe;
 
-                decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+                decodedSize = decoder((const char*)selectedIn, (char*)dstPtr,
+                        (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
+                        (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
                 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC);   /* decompression failed */
-                if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
-                if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
+                if (dctxPtr->frameInfo.contentChecksumFlag)
+                    XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
+                if (dctxPtr->frameInfo.contentSize)
+                    dctxPtr->frameRemainingSize -= decodedSize;
 
                 /* dictionary management */
                 if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
@@ -1284,10 +1299,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 }
 
                 /* Decode */
-                decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
-                if (decodedSize < 0) return err0r(LZ4F_ERROR_decompressionFailed);   /* decompression failed */
-                if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
-                if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize;
+                decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut,
+                        (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
+                        (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+                if (decodedSize < 0)
+                    return err0r(LZ4F_ERROR_decompressionFailed);   /* decompression failed */
+                if (dctxPtr->frameInfo.contentChecksumFlag)
+                    XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
+                if (dctxPtr->frameInfo.contentSize)
+                    dctxPtr->frameRemainingSize -= decodedSize;
                 dctxPtr->tmpOutSize = decodedSize;
                 dctxPtr->tmpOutStart = 0;
                 dctxPtr->dStage = dstage_flushOut;
@@ -1318,7 +1338,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
 
         case dstage_getSuffix:
             {   size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
-                if (dctxPtr->frameRemainingSize) return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
+                if (dctxPtr->frameRemainingSize)
+                    return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
                 if (suffixSize == 0) {  /* frame completed */
                     nextSrcSizeHint = 0;
                     dctxPtr->dStage = dstage_getHeader;
@@ -1379,7 +1400,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
                 dctxPtr->tmpInSize += sizeToCopy;
-                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */
+                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
+                    /* not enough input to get full sBlockSize; wait for more */
                     nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
                     doAnotherStage = 0;
                     break;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 9a2130a..b1719e2 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -364,14 +364,11 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
 
 /*! LZ4F_resetDecompressionContext() : v1.8.0
- *  When decompression ends successfully,
- *  it's possible to start a new decompression immediately
- *  re-using the same context.
- *  However, in case of an error, the context is left in "undefined" state.
+ *  In case of an error, the context is left in "undefined" state.
  *  In which case, it's necessary to reset it, before re-using it.
  *  This method can also be used to abruptly stop an unfinished decompression,
- *  and start a new on the same context. */
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
+ *  and start a new with the same context. */
+LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
 
 
 
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 8ea496d..d3bae82 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -50,15 +50,6 @@ extern "C" {
 #include "lz4frame.h"
 
 
-/* ---   Experimental functions   --- */
-/* LZ4F_resetDecompressionContext() :
- * LZ4F_decompress() does not guarantee to leave dctx in clean state in case of errors.
- * In order to re-use a dctx after a decompression error,
- * use LZ4F_resetDecompressionContext() first.
- * dctx will be able to start decompression on a new frame */
-LZ4FLIB_API LZ4F_errorCode_t LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);
-
-
 /* ---   Error List   --- */
 #define LZ4F_LIST_ERRORS(ITEM) \
         ITEM(OK_NoError) \
diff --git a/tests/frametest.c b/tests/frametest.c
index 04597a4..538bedc 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -40,6 +40,7 @@
 #include       /* fprintf */
 #include      /* strcmp */
 #include        /* clock_t, clock(), CLOCKS_PER_SEC */
+#include 
 #include "lz4frame_static.h"
 #include "lz4.h"        /* LZ4_VERSION_STRING */
 #define XXH_STATIC_LINKING_ONLY
@@ -67,7 +68,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
 #define GB *(1U<<30)
 
 static const U32 nbTestsDefault = 256 KB;
-#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime1 = 2654435761U;
 static const U32 prime2 = 2246822519U;
@@ -166,7 +166,7 @@ static unsigned FUZ_highbit(U32 v32)
 *********************************************************/
 int basicTests(U32 seed, double compressibility)
 {
-    int testResult = 0;
+#define COMPRESSIBLE_NOISE_LENGTH (2 MB)
     void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
     size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
     void* const compressedBuffer = malloc(cBuffSize);
@@ -176,9 +176,10 @@ int basicTests(U32 seed, double compressibility)
     LZ4F_decompressionContext_t dCtx = NULL;
     LZ4F_compressionContext_t cctx = NULL;
     U64 crcOrig;
-
+    int basicTests_error = 0;
     LZ4F_preferences_t prefs;
     memset(&prefs, 0, sizeof(prefs));
+
     if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
         DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
         goto _output_error;
@@ -198,7 +199,7 @@ int basicTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
     cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
     if (LZ4F_isError(cSize)) goto _output_error;
-    DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
+    DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
 
     DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
     { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
@@ -236,8 +237,6 @@ int basicTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "Decompression test : \n");
     {   size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
         size_t compressedBufferSize = cSize;
-        BYTE* ip = (BYTE*)compressedBuffer;
-        BYTE* const iend = (BYTE*)compressedBuffer + cSize;
 
         LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
         if (LZ4F_isError(errorCode)) goto _output_error;
@@ -280,6 +279,7 @@ int basicTests(U32 seed, double compressibility)
         {   size_t oSize = 0;
             size_t iSize = 0;
             LZ4F_frameInfo_t fi;
+            const BYTE* ip = (BYTE*)compressedBuffer;
 
             DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
             errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
@@ -315,10 +315,30 @@ int basicTests(U32 seed, double compressibility)
             ip += iSize;
         }
 
+        DISPLAYLEVEL(3, "Decode a buggy input : ");
+        assert(COMPRESSIBLE_NOISE_LENGTH > 64);
+        assert(cSize > 48);
+        memcpy(decodedBuffer, (char*)compressedBuffer+16, 32);  /* save correct data */
+        memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32);  /* insert noise */
+        {   size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
+            size_t cbSize = cSize;
+            size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
+                                                               compressedBuffer, &cbSize,
+                                                               NULL);
+            if (!LZ4F_isError(decompressError)) goto _output_error;
+            DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
+        }
+        memcpy((char*)compressedBuffer+16, decodedBuffer, 32);  /* restore correct data */
+
+        DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
+        LZ4F_resetDecompressionContext(dCtx);   /* always successful */
+
         DISPLAYLEVEL(3, "Byte after byte : ");
         {   BYTE* const ostart = (BYTE*)decodedBuffer;
             BYTE* op = ostart;
             BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
+            const BYTE* ip = compressedBuffer;
+            const BYTE* const iend = ip + cSize;
             while (ip < iend) {
                 size_t oSize = oend-op;
                 size_t iSize = 1;
@@ -330,11 +350,7 @@ int basicTests(U32 seed, double compressibility)
             { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
               if (crcDest != crcOrig) goto _output_error; }
             DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH);
-            }
-
-        errorCode = LZ4F_freeDecompressionContext(dCtx);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        dCtx = NULL;
+        }
     }
 
     DISPLAYLEVEL(3, "Using 64 KB block : ");
@@ -366,9 +382,6 @@ int basicTests(U32 seed, double compressibility)
         const BYTE* ip = (const BYTE*)compressedBuffer;
         const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
 
-        { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
-          if (LZ4F_isError(createError)) goto _output_error; }
-
         DISPLAYLEVEL(3, "random segment sizes : ");
         while (ip < iend) {
             unsigned const nbBits = FUZ_rand(&randState) % maxBits;
@@ -552,10 +565,10 @@ _end:
     free(decodedBuffer);
     LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
     LZ4F_freeCompressionContext(cctx); cctx = NULL;
-    return testResult;
+    return basicTests_error;
 
 _output_error:
-    testResult = 1;
+    basicTests_error = 1;
     DISPLAY("Error detected ! \n");
     goto _end;
 }
-- 
cgit v0.12


From 2600a154bebdf79637e289ed19e53e2c72a168d7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 10 May 2017 17:06:31 -0700
Subject: fix (minor) g++ compatibility for frametest

---
 tests/frametest.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/frametest.c b/tests/frametest.c
index 538bedc..0dadf9f 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -337,7 +337,7 @@ int basicTests(U32 seed, double compressibility)
         {   BYTE* const ostart = (BYTE*)decodedBuffer;
             BYTE* op = ostart;
             BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
-            const BYTE* ip = compressedBuffer;
+            const BYTE* ip = (const BYTE*) compressedBuffer;
             const BYTE* const iend = ip + cSize;
             while (ip < iend) {
                 size_t oSize = oend-op;
-- 
cgit v0.12


From 4e359f5c35d6c2df3051a9fad64f6b19ac6a918b Mon Sep 17 00:00:00 2001
From: PierreNav 
Date: Tue, 23 May 2017 11:16:15 +0200
Subject: Add DLL files to the INSTALL target

---
 contrib/cmake_unofficial/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt
index e59a756..27c3a78 100644
--- a/contrib/cmake_unofficial/CMakeLists.txt
+++ b/contrib/cmake_unofficial/CMakeLists.txt
@@ -169,7 +169,8 @@ if(NOT LZ4_BUNDLED_MODE)
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
   install(TARGETS ${LZ4_LIBRARIES_BUILT}
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
   install(FILES
     "${LZ4_LIB_SOURCE_DIR}/lz4.h"
     "${LZ4_LIB_SOURCE_DIR}/lz4frame.h"
-- 
cgit v0.12


From 1c9d4091dab0b27f63d067b6be5701afb9aba124 Mon Sep 17 00:00:00 2001
From: Chocobo1 
Date: Fri, 26 May 2017 01:03:05 +0800
Subject: [Doc] Fix markdown

---
 doc/lz4_Frame_format.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 2ea1a86..5f8f479 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -1,7 +1,7 @@
 LZ4 Frame Format Description
 ============================
 
-###Notices
+### Notices
 
 Copyright (c) 2013-2015 Yann Collet
 
@@ -14,7 +14,7 @@ and that any substantive changes or deletions from the original
 are clearly marked.
 Distribution of this document is unlimited.
 
-###Version
+### Version
 
 1.5.1 (31/03/2015)
 
-- 
cgit v0.12


From 03d8586fca653cb10379a6a4f310b0724d9ac6ac Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 26 May 2017 14:38:47 -0700
Subject: cli accept block sizes with KB / MB prefixes

---
 programs/lz4cli.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 10aa5aa..f3583ef 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -258,14 +258,22 @@ static int exeNameMatch(const char* exeName, const char* test)
 }
 
 /*! readU32FromChar() :
-    @return : unsigned integer value reach from input in `char` format
+    @return : unsigned integer value read from input in `char` format
+    allows and interprets K, KB, KiB, M, MB and MiB suffix.
     Will also modify `*stringPtr`, advancing it to position where it stopped reading.
-    Note : this function can overflow if result > MAX_UINT */
+    Note : function result can overflow if digit string > MAX_UINT */
 static unsigned readU32FromChar(const char** stringPtr)
 {
     unsigned result = 0;
     while ((**stringPtr >='0') && (**stringPtr <='9'))
         result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        result <<= 10;
+        if (**stringPtr=='M') result <<= 10;
+        (*stringPtr)++ ;
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
     return result;
 }
 
-- 
cgit v0.12


From 7e15e240aba842020a2f6e86f35e71cbacdf237d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 6 Jun 2017 11:20:36 -0700
Subject: added a paragraph on overlap matches

---
 doc/lz4_Block_format.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md
index 0f6a5ba..4e39b41 100644
--- a/doc/lz4_Block_format.md
+++ b/doc/lz4_Block_format.md
@@ -90,10 +90,18 @@ A 255 value means there is another byte to read and add.
 There is no limit to the number of optional bytes that can be output this way.
 (This points towards a maximum achievable compression ratio of about 250).
 
-With the offset and the matchlength,
-the decoder can now proceed to copy the data from the already decoded buffer.
-On decoding the matchlength, we reach the end of the compressed sequence,
-and therefore start another one.
+Decoding the matchlength reaches the end of current sequence.
+Next byte will be the start of another sequence.
+But before moving to next sequence,
+it's time to use the decoded match position and length.
+The decoder copies matchlength bytes from match position to current position.
+
+In some cases, matchlength is larger than offset.
+Therefore, match pos + match length > current pos,
+which means that later bytes to copy are not yet decoded.
+This is called an "overlap match", and must be handled with special care.
+The most common case is an offset of 1,
+meaning the last byte is repeated matchlength times.
 
 
 Parsing restrictions
-- 
cgit v0.12


From 775e63ee01af461d20a607f658ca3795ff738157 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 8 Jun 2017 12:51:56 -0700
Subject: refactored simple_buffer.c example (#363)

---
 examples/.gitignore      |  1 +
 examples/Makefile        | 11 ++++---
 examples/simple_buffer.c | 81 +++++++++++++++++++++++++-----------------------
 3 files changed, 50 insertions(+), 43 deletions(-)

diff --git a/examples/.gitignore b/examples/.gitignore
index 3ceb90d..5abeef6 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -6,4 +6,5 @@
 /ringBufferHC
 /lineCompress
 /frameCompress
+/simpleBuffer
 /*.exe
diff --git a/examples/Makefile b/examples/Makefile
index aad713b..7647f16 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -27,9 +27,10 @@
 # kindly provided by Takayuki Matsuoka
 # ##########################################################################
 
-CFLAGS ?= -O3
-CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
-FLAGS  := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+CPPFLAGS += -I../lib
+CFLAGS   ?= -O3
+CFLAGS   += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
+FLAGS    := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 TESTFILE= Makefile
 LZ4DIR := ../lib
@@ -48,7 +49,8 @@ endif
 
 default: all
 
-all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC lineCompress frameCompress
+all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
+     lineCompress frameCompress simpleBuffer
 
 printVersion: $(LZ4DIR)/lz4.c printVersion.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
@@ -85,6 +87,7 @@ test : all
 	./ringBufferHC$(EXT) $(TESTFILE)
 	./lineCompress$(EXT) $(TESTFILE)
 	./frameCompress$(EXT) $(TESTFILE)
+	./simpleBuffer$(EXT)
 	$(LZ4) -vt $(TESTFILE).lz4
 
 clean:
diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index f531d48..ee29148 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -8,10 +8,10 @@
  */
 
 /* Includes, for Power! */
-#include "lz4.h"    // This is all that is required to expose the prototypes for basic compression and decompression.
-#include   // For printf()
-#include  // For memcmp()
-#include  // For exit()
+#include "lz4.h"     // This is all that is required to expose the prototypes for basic compression and decompression.
+#include    // For printf()
+#include   // For memcmp()
+#include   // For exit()
 
 /*
  * Easy show-error-and-bail function.
@@ -28,37 +28,39 @@ void run_screaming(const char *message, const int code) {
  */
 int main(void) {
   /* Introduction */
-  // Below we will have a Compression and Decompression section to demonstrate.  There are a few important notes before we start:
-  //   1) The return codes of LZ4_ functions are important.  Read lz4.h if you're unsure what a given code means.
-  //   2) LZ4 uses char* pointers in all LZ4_ functions.  This is baked into the API and probably not going to change.  If your
-  //      program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the
-  //      right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
+  // Below we will have a Compression and Decompression section to demonstrate.
+  // There are a few important notes before we start:
+  //   1) The return codes of LZ4_ functions are important.
+  //      Read lz4.h if you're unsure what a given code means.
+  //   2) LZ4 uses char* pointers in all LZ4_ functions.
+  //      This is baked into the API and probably not going to change.
+  //      If your program uses pointers that are unsigned char*, void*, or otherwise different,
+  //      you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
 
   /* Compression */
   // We'll store some text into a variable pointed to by *src to be compressed later.
-  const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
-  // The compression function needs to know how many bytes of exist.  Since we're using a string, we can use strlen() + 1 (for \0).
-  const size_t src_size = strlen(src) + 1;
+  const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
+  // The compression function needs to know how many bytes exist.  Since we're using a string, we can use strlen() + 1 (for \0).
+  const int src_size = (int)(strlen(src) + 1);
   // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
-  const size_t max_dst_size = LZ4_compressBound(src_size);
+  const int max_dst_size = LZ4_compressBound(src_size);
   // We will use that size for our destination boundary when allocating space.
-  char *compressed_data = malloc(max_dst_size);
+  char* compressed_data = malloc(max_dst_size);
   if (compressed_data == NULL)
     run_screaming("Failed to allocate memory for *compressed_data.", 1);
-  // That's all the information and preparation LZ4 needs to compress *src into *compressed_data.  Invoke LZ4_compress_default now
-  // with our size values and pointers to our memory locations.  Save the return value for error checking.
-  int return_value = 0;
-  return_value = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
+  // That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
+  // Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
+  // Save the return value for error checking.
+  const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
   // Check return_value to determine what happened.
-  if (return_value < 0)
-    run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data.  See exit code (echo $?) for value returned.", return_value);
-  if (return_value == 0)
+  if (compressed_data_size < 0)
+    run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data.  See exit code (echo $?) for value returned.", compressed_data_size);
+  if (compressed_data_size == 0)
     run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1);
-  if (return_value > 0)
+  if (compressed_data_size > 0)
     printf("We successfully compressed some data!\n");
-  // Not only does a positive return_value mean success, the value returned == the number of bytes required.  You can use this to
-  // realloc() *compress_data to free up memory, if desired.  We'll do so just to demonstrate the concept.
-  const size_t compressed_data_size = return_value;
+  // Not only does a positive return_value mean success, the value returned == the number of bytes required.
+  // You can use this to realloc() *compress_data to free up memory, if desired.  We'll do so just to demonstrate the concept.
   compressed_data = (char *)realloc(compressed_data, compressed_data_size);
   if (compressed_data == NULL)
     run_screaming("Failed to re-alloc memory for compressed_data.  Sad :(", 1);
@@ -66,25 +68,26 @@ int main(void) {
   /* Decompression */
   // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite!  We'll create a
   // *new_src location of size src_size since we know that value.
-  char *new_src = malloc(src_size);
-  if (new_src == NULL)
-    run_screaming("Failed to allocate memory for *new_src.", 1);
-  // The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src
-  // memory location is, and how large the new_src (uncompressed) output will be.  Again, save the return_value.
-  return_value = LZ4_decompress_safe(compressed_data, new_src, compressed_data_size, src_size);
-  if (return_value < 0)
-    run_screaming("A negative result from LZ4_decompress_fast indicates a failure trying to decompress the data.  See exit code (echo $?) for value returned.", return_value);
-  if (return_value == 0)
+  char* const regen_buffer = malloc(src_size);
+  if (regen_buffer == NULL)
+    run_screaming("Failed to allocate memory for *regen_buffer.", 1);
+  // The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is,
+  // where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be.
+  // Again, save the return_value.
+  const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size);
+  if (decompressed_size < 0)
+    run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data.  See exit code (echo $?) for value returned.", decompressed_size);
+  if (decompressed_size == 0)
     run_screaming("I'm not sure this function can ever return 0.  Documentation in lz4.h doesn't indicate so.", 1);
-  if (return_value > 0)
+  if (decompressed_size > 0)
     printf("We successfully decompressed some data!\n");
-  // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data
-  // stream.  I'm not sure there's ever a time you'll need to know this in most cases...
+  // Not only does a positive return value mean success,
+  // value returned == number of bytes regenerated from compressed_data stream.
 
   /* Validation */
   // We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
-  if (memcmp(src, new_src, src_size) != 0)
+  if (memcmp(src, regen_buffer, src_size) != 0)
     run_screaming("Validation failed.  *src and *new_src are not identical.", 1);
-  printf("Validation done.  The string we ended up with is:\n%s\n", new_src);
+  printf("Validation done.  The string we ended up with is:\n%s\n", regen_buffer);
   return 0;
 }
-- 
cgit v0.12


From 0beaa356f26aa5ee9d37673a5e72d8bc751f69c2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 8 Jun 2017 15:04:55 -0700
Subject: fixed minor scan-build warning

---
 examples/simple_buffer.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c
index ee29148..403d9e8 100644
--- a/examples/simple_buffer.c
+++ b/examples/simple_buffer.c
@@ -75,6 +75,7 @@ int main(void) {
   // where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be.
   // Again, save the return_value.
   const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size);
+  free(compressed_data);   /* no longer useful */
   if (decompressed_size < 0)
     run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data.  See exit code (echo $?) for value returned.", decompressed_size);
   if (decompressed_size == 0)
-- 
cgit v0.12


From 139a387d179dfac91dbaeef994bbbea4d79eb9ba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 12 Jun 2017 10:54:59 -0700
Subject: updated NEWS

---
 NEWS        | 3 ++-
 lib/lz4hc.h | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 32fb613..ab95149 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,8 @@
-v1.7.6
+v1.8.0
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
 cli : added GNU separator -- specifying that all following arguments are files
 API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
+API : added LZ4F_resetDecompressionContext()
 API : lz4frame : negative compression levels trigger fast acceleration
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : dragonFlyBSD, OpenBSD, NetBSD supported
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index c4284c0..b63c825 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -237,9 +237,9 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStr
 #ifndef LZ4_HC_SLO_098092834
 #define LZ4_HC_SLO_098092834
 
-/*! LZ4_compress_HC_destSize() :
+/*! LZ4_compress_HC_destSize() : v1.8.0 (experimental)
  *  Will try to compress as much data from `src` as possible
- *  that can fit in `targetDstSize` budget.
+ *  that can fit into `targetDstSize` budget.
  *  Result is provided in 2 parts :
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
-- 
cgit v0.12


From d18084c65561cddec48f74c3a91363c9f8e6f709 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 12 Jun 2017 15:47:43 -0700
Subject: report where decompression ends (#313)

suggested by @ehem
note : only works for files < 2 GB
---
 programs/lz4io.c | 99 ++++++++++++++++++++++++++++++++++----------------------
 tests/Makefile   |  3 ++
 2 files changed, 63 insertions(+), 39 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index e588f7a..1e6b437 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -218,7 +218,9 @@ void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
 ** ************************************************************************ */
 
 static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
-static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; }
+static int LZ4IO_isSkippableMagicNumber(unsigned int magic) {
+    return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0;
+}
 
 
 /** LZ4IO_openSrcFile() :
@@ -321,7 +323,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
 
     /* Init */
     clock_t const clockStart = clock();
-    if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC;
+    compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
 
     finput = LZ4IO_openSrcFile(input_filename);
     if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename);
@@ -350,12 +352,14 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
         /* Compress Block */
         outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
         compressedfilesize += outSize+4;
-        DISPLAYUPDATE(2, "\rRead : %i MB  ==> %.2f%%   ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+        DISPLAYUPDATE(2, "\rRead : %i MB  ==> %.2f%%   ",
+                (int)(filesize>>20), (double)compressedfilesize/filesize*100);
 
         /* Write Block */
         LZ4IO_writeLE32(out_buff, outSize);
         {   size_t const sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
-            if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(24, "Write error : cannot write compressed block");
+            if (sizeCheck!=(size_t)(outSize+4))
+                EXM_THROW(24, "Write error : cannot write compressed block");
     }   }
     if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
 
@@ -367,7 +371,8 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output
     DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
         filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
     {   double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
-        DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+        DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds,
+                        (double)filesize / seconds / 1024 / 1024);
     }
 
     /* Close & Free */
@@ -870,22 +875,24 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
 {
     unsigned char MNstore[MAGICNUMBER_SIZE];
     unsigned magicNumber;
-    static unsigned nbCalls = 0;
+    static unsigned nbFrames = 0;
 
     /* init */
-    nbCalls++;
+    nbFrames++;
 
     /* Check Archive Header */
     if (g_magicRead) {  /* magic number already read from finput (see legacy frame)*/
-      magicNumber = g_magicRead;
-      g_magicRead = 0;
+        magicNumber = g_magicRead;
+        g_magicRead = 0;
     } else {
-      size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
-      if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; }   /* EOF */
-      if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
-      magicNumber = LZ4IO_readLE32(MNstore);   /* Little Endian format */
+        size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
+        if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; }   /* EOF */
+        if (nbReadBytes != MAGICNUMBER_SIZE)
+          EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
+        magicNumber = LZ4IO_readLE32(MNstore);   /* Little Endian format */
     }
-    if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0;  /* fold skippable magic numbers */
+    if (LZ4IO_isSkippableMagicNumber(magicNumber))
+        magicNumber = LZ4IO_SKIPPABLE0;   /* fold skippable magic numbers */
 
     switch(magicNumber)
     {
@@ -896,22 +903,32 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
         return LZ4IO_decodeLegacyStream(finput, foutput);
     case LZ4IO_SKIPPABLE0:
         DISPLAYLEVEL(4, "Skipping detected skippable area \n");
-        { size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
-          if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); }
-        { unsigned const size = LZ4IO_readLE32(MNstore);     /* Little Endian format */
-          int const errorNb = fseek_u32(finput, size, SEEK_CUR);
-          if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); }
+        {   size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
+            if (nbReadBytes != 4)
+                EXM_THROW(42, "Stream error : skippable size unreadable");
+        }
+        {   unsigned const size = LZ4IO_readLE32(MNstore);
+            int const errorNb = fseek_u32(finput, size, SEEK_CUR);
+            if (errorNb != 0)
+                EXM_THROW(43, "Stream error : cannot skip skippable area");
+        }
         return 0;
     EXTENDED_FORMAT;  /* macro extension for custom formats */
     default:
-        if (nbCalls == 1) {  /* just started */
+        if (nbFrames == 1) {  /* just started */
+            /* Wrong magic number at the beginning of 1st stream */
             if (!g_testMode && g_overwrite) {
-                nbCalls = 0;
+                nbFrames = 0;
                 return LZ4IO_passThrough(finput, foutput, MNstore);
             }
-            EXM_THROW(44,"Unrecognized header : file cannot be decoded");   /* Wrong magic number at the beginning of 1st stream */
+            EXM_THROW(44,"Unrecognized header : file cannot be decoded");
+        }
+        {   long int const position = ftell(finput);  /* only works for files < 2 GB */
+            DISPLAYLEVEL(2, "Stream followed by undecodable data ");
+            if (position != -1L)
+                DISPLAYLEVEL(2, "at position %i ", (int)position);
+            DISPLAYLEVEL(2, "\n");
         }
-        DISPLAYLEVEL(2, "Stream followed by undecodable data\n");
         return ENDOFSTREAM;
     }
 }
@@ -920,23 +937,26 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu
 static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename)
 {
     FILE* const foutput = ress.dstFile;
-    unsigned long long filesize = 0, decodedSize=0;
+    unsigned long long filesize = 0;
 
     /* Init */
     FILE* const finput = LZ4IO_openSrcFile(input_filename);
     if (finput==NULL) return 1;
 
     /* Loop over multiple streams */
-    do {
-        decodedSize = selectDecoder(ress, finput, foutput);
-        if (decodedSize != ENDOFSTREAM)
-            filesize += decodedSize;
-    } while (decodedSize != ENDOFSTREAM);
+    for ( ; ; ) {  /* endless loop, see break condition */
+        unsigned long long const decodedSize =
+                        selectDecoder(ress, finput, foutput);
+        if (decodedSize == ENDOFSTREAM) break;
+        filesize += decodedSize;
+    }
 
-    /* Close */
+    /* Close input */
     fclose(finput);
-
-    if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); }  /* remove source file : --rm */
+    if (g_removeSrcFile) {  /* --rm */
+        if (remove(input_filename))
+            EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
+    }
 
     /* Final Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
@@ -949,7 +969,7 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
 
 static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
 {
-    FILE* const foutput = LZ4IO_openDstFile(output_filename);   /* success or die */
+    FILE* const foutput = LZ4IO_openDstFile(output_filename);
     if (foutput==NULL) return 1;   /* failure */
 
     ress.dstFile = foutput;
@@ -959,12 +979,13 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con
 
     /* Copy owner, file permissions and modification time */
     {   stat_t statbuf;
-        if (strcmp (input_filename, stdinmark)
-            && strcmp (output_filename, stdoutmark)
-            && strcmp (output_filename, nulmark)
-            && UTIL_getFileStat(input_filename, &statbuf))
-            UTIL_setFileStat(output_filename, &statbuf);  /* should return value be read ? or is silent fail good enough ? */
-    }
+        if ( strcmp (input_filename, stdinmark)
+          && strcmp (output_filename, stdoutmark)
+          && strcmp (output_filename, nulmark)
+          && UTIL_getFileStat(input_filename, &statbuf) ) {
+            UTIL_setFileStat(output_filename, &statbuf);
+            /* should return value be read ? or is silent fail good enough ? */
+    }   }
 
     return 0;
 }
diff --git a/tests/Makefile b/tests/Makefile
index bc22423..07bf401 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -260,6 +260,9 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	$(LZ4) -d --rm -- -z tmp4          # uncompresses ./-z into tmp4
 	test ! -f ./-z
 	$(DIFF) -q tmp tmp4
+	$(LZ4) -f tmp
+	cat tmp >> tmp.lz4
+	$(LZ4) -f tmp.lz4         # uncompress valid frame followed by invalid data
 	@$(RM) tmp*
 
 test-lz4-hugefile: lz4 datagen
-- 
cgit v0.12


From bfc18066053a26b5acc9eb286bfe6b2a87b14318 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 13 Jun 2017 12:35:00 -0700
Subject: clarified lz4frame api comment (#350)

---
 lib/lz4frame.h | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index b1719e2..dd2be58 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -287,7 +287,7 @@ typedef struct LZ4F_dctx_s LZ4F_dctx;   /* incomplete type */
 typedef LZ4F_dctx* LZ4F_decompressionContext_t;   /* compatibility with previous API versions */
 
 typedef struct {
-  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
   unsigned reserved[3];
 } LZ4F_decompressOptions_t;
 
@@ -295,9 +295,9 @@ typedef struct {
 /* Resource management */
 
 /*!LZ4F_createDecompressionContext() :
- * Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
- * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+ * Create an LZ4F_dctx object, to track all decompression operations.
+ * The version provided MUST be LZ4F_VERSION.
+ * The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  * The result is an errorCode, which can be tested using LZ4F_isError().
  * dctx memory can be released using LZ4F_freeDecompressionContext();
  * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
@@ -313,20 +313,22 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
 /*! LZ4F_getFrameInfo() :
  * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional. Extracted information can be useful for allocation purposes, typically.
+ * Its usage is optional.
+ * Extracted information can typically be useful for allocation purposes.
  * This function works in 2 situations :
- *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+ *   - At the beginning of a new frame, in which case
+ *     it will decode information from `srcBuffer`, starting the decoding process.
  *     Input size must be large enough to successfully decode the entire frame header.
  *     Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
  *     It's allowed to provide more input data than this minimum.
  *   - After decoding has been started.
  *     In which case, no input is read, frame parameters are extracted from dctx.
- *     If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
+ *   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
  * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
  * Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
  *           or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
  * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  */
 LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
@@ -334,18 +336,18 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
 /*! LZ4F_decompress() :
- * Call this function repetitively to regenerate data compressed within `srcBuffer`.
+ * Call this function repetitively to regenerate compressed data from `srcBuffer`.
  * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
  *
- * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
  *
- * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
+ * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ * Number of bytes consumed can be < number of bytes provided.
  * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Remaining data will have to be presented again in a subsequent invocation.
+ * Unconsumed source data must be presented again in subsequent invocations.
  *
  * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- * `dstBuffer` can be changed at will between each consecutive function invocation.
+ * `dstBuffer` itself can be changed at will between each consecutive function invocation.
  *
  * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
@@ -367,7 +369,7 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
  *  In case of an error, the context is left in "undefined" state.
  *  In which case, it's necessary to reset it, before re-using it.
  *  This method can also be used to abruptly stop an unfinished decompression,
- *  and start a new with the same context. */
+ *  and start a new one using the same context. */
 LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
 
 
-- 
cgit v0.12


From a6fd0f9d0ba4dd3db5ffd221a6b3384296a8deed Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 13 Jun 2017 14:45:48 -0700
Subject: -g compilation flag not by default for lz4 cli

---
 NEWS              | 3 +--
 programs/Makefile | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index ab95149..f6df47c 100644
--- a/NEWS
+++ b/NEWS
@@ -3,13 +3,12 @@ cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
 cli : added GNU separator -- specifying that all following arguments are files
 API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
 API : added LZ4F_resetDecompressionContext()
-API : lz4frame : negative compression levels trigger fast acceleration
+API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
 doc : lz4 api manual, by Przemyslaw Skibinski
 
-
 v1.7.5
 lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski
 lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z)
diff --git a/programs/Makefile b/programs/Makefile
index 2e73780..8cf7d59 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -48,7 +48,7 @@ VOID := /dev/null
 
 CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 CFLAGS  ?= -O3
-DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-- 
cgit v0.12


From 89b9f026c913c0d3e9995f52b49b37f1de1af150 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 13 Jun 2017 17:25:29 -0700
Subject: made level 10 a bit faster

at the expense of a little bit of compression ratio.

Now speed is intermediate on calgary corpus :
25 - 12 - 8 - 3
---
 lib/lz4hc.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b987d39..a423aea 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -182,7 +182,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const dictBase = hc4->dictBase;
     int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
-    U32   matchIndex;
+    U32 matchIndex;
 
 
     /* First Match */
@@ -192,16 +192,17 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
         nbAttempts--;
         if (matchIndex >= dictLimit) {
-            const BYTE* matchPtr = base + matchIndex;
+            const BYTE* const matchPtr = base + matchIndex;
             if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
                 if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
                     int back = 0;
 
-                    while ((ip+back > iLowLimit)
-                           && (matchPtr+back > lowPrefixPtr)
-                           && (ip[back-1] == matchPtr[back-1]))
+                    while ( (ip+back > iLowLimit)
+                         && (matchPtr+back > lowPrefixPtr)
+                         && (ip[back-1] == matchPtr[back-1])) {
                             back--;
+                    }
 
                     mlt -= back;
 
@@ -548,7 +549,7 @@ static int LZ4HC_compress_generic (
         if (limit == limitedDestSize) cLevel = 10;
         switch (cLevel) {
             case 10:
-                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (15-1), limit);
+                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << 12, limit);
             case 11:
                 ctx->searchNum = LZ4HC_getSearchNum(cLevel);
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
-- 
cgit v0.12


From a4ec7fa7b7863e870d3b2ea8d2173083ae95ee95 Mon Sep 17 00:00:00 2001
From: Chocobo1 
Date: Mon, 19 Jun 2017 13:09:36 +0800
Subject: Fix gcc7 Wimplicit-fallthrough warnings

For the default Wimplicit-fallthrough=3 level,
the comment should start with "fall*"
---
 lib/lz4frame.c    | 6 +++---
 lib/lz4hc.c       | 2 +-
 programs/lz4cli.c | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index fb37789..994cc8b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1096,7 +1096,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
             if (srcEnd-srcPtr == 0) return minFHSize;   /* 0-size input */
             dctxPtr->tmpInTarget = minFHSize;   /* minimum to attempt decode */
             dctxPtr->dStage = dstage_storeHeader;
-            /* pass-through */
+            /* fall-through */
 
         case dstage_storeHeader:
             {   size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
@@ -1138,7 +1138,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
             dctxPtr->tmpOutSize = 0;
 
             dctxPtr->dStage = dstage_getCBlockSize;
-            /* pass-through */
+            /* fall-through */
 
         case dstage_getCBlockSize:
             if ((size_t)(srcEnd - srcPtr) >= BHSize) {
@@ -1236,8 +1236,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 }
                 selectedIn = dctxPtr->tmpIn;
                 dctxPtr->dStage = dstage_decodeCBlock;
-                /* pass-through */
             }
+            /* fall-through */
 
         case dstage_decodeCBlock:
             if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize)   /* not enough place into dst : decode into tmpOut */
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a423aea..ca9c2e6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -555,7 +555,7 @@ static int LZ4HC_compress_generic (
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
             default:
                 cLevel = 12;
-                /* pass-through */
+                /* fall-through */
             case 12:
                 ctx->searchNum = LZ4HC_getSearchNum(cLevel);
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1);
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index f3583ef..55f3133 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -469,8 +469,9 @@ int main(int argc, const char** argv)
 
 #ifdef UTIL_HAS_CREATEFILELIST
                     /* recursive */
-                case 'r': recursive=1;  /* without break */
+                case 'r': recursive=1;
 #endif
+                    /* fall-through */
                     /* Treat non-option args as input files.  See https://code.google.com/p/lz4/issues/detail?id=151 */
                 case 'm': multiple_inputs=1;
                     break;
-- 
cgit v0.12


From 9d453a1885c17376fbe42924ad4ad6e2913b13ad Mon Sep 17 00:00:00 2001
From: Chocobo1 
Date: Mon, 19 Jun 2017 14:14:52 +0800
Subject: Update README.md

Update appveyor badge URL
Remove empty line
---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 04ba0db..950e4c4 100644
--- a/README.md
+++ b/README.md
@@ -25,14 +25,13 @@ LZ4 library is provided as open-source software using BSD 2-Clause license.
 [travisMasterBadge]: https://travis-ci.org/lz4/lz4.svg?branch=master "Continuous Integration test suite"
 [travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.org/lz4/lz4
-[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/master?svg=true "Windows test suite"
-[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/jc2yhgwyc7qqtsko/branch/dev?svg=true "Windows test suite"
+[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=master&svg=true "Windows test suite"
+[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite"
 [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh
 [coverBadge]: https://scan.coverity.com/projects/4735/badge.svg "Static code analysis of Master branch"
 [coverlink]: https://scan.coverity.com/projects/4735
 
 > **Branch Policy:**
-
 > - The "master" branch is considered stable, at all times.
 > - The "dev" branch is the one where all contributions must be merged
     before being promoted to master.
-- 
cgit v0.12


From e14b4c5a3eceb8ea9490b29031a94863187062ed Mon Sep 17 00:00:00 2001
From: Jussi Kukkonen 
Date: Mon, 26 Jun 2017 12:07:09 +0300
Subject: tests/Makefile: don't use LIBDIR as variable

LIBDIR may be overriden with a environment variable: In this case make
clean breaks in tests/. Use another variable name.
---
 tests/Makefile | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 07bf401..5d2532f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -34,7 +34,7 @@ DESTDIR ?=
 PREFIX  ?= /usr/local
 BINDIR  := $(PREFIX)/bin
 MANDIR  := $(PREFIX)/share/man/man1
-LIBDIR  := ../lib
+LZ4DIR  := ../lib
 PRGDIR  := ../programs
 VOID    := /dev/null
 TESTDIR := versionsTest
@@ -45,7 +45,7 @@ CFLAGS  += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-e
            -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(MOREFLAGS)
-CPPFLAGS:= -I$(LIBDIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
+CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
 FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 
@@ -81,31 +81,31 @@ lz4c32:   # create a 32-bits version for 32/64 interop tests
 	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
 	cp $(LZ4) $(LZ4)c32
 
-fullbench  : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/lz4frame.o $(LIBDIR)/xxhash.o fullbench.c
+fullbench  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-fullbench-lib: fullbench.c $(LIBDIR)/xxhash.c
-	$(MAKE) -C $(LIBDIR) liblz4.a
-	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4.a
+fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c
+	$(MAKE) -C $(LZ4DIR) liblz4.a
+	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
 
-fullbench-dll: fullbench.c $(LIBDIR)/xxhash.c
-	$(MAKE) -C $(LIBDIR) liblz4
-	$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LIBDIR)/dll/liblz4.dll
+fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
+	$(MAKE) -C $(LZ4DIR) liblz4
+	$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
 
-fuzzer  : $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o fuzzer.c
+fuzzer  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frametest: $(LIBDIR)/lz4frame.o $(LIBDIR)/lz4.o $(LIBDIR)/lz4hc.o $(LIBDIR)/xxhash.o frametest.c
+frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-fasttest: $(LIBDIR)/lz4.o fasttest.c
+fasttest: $(LZ4DIR)/lz4.o fasttest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
 clean:
-	@$(MAKE) -C $(LIBDIR) $@ > $(VOID)
+	@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(RM) core *.o *.test tmp* \
         fullbench-dll$(EXT) fullbench-lib$(EXT) \
-- 
cgit v0.12


From 6ad3a983dbf3a25273e04051d29022c72c469cd5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 26 Jun 2017 11:29:05 -0700
Subject: fix #369

The bug would make the bt search read one byte in an invalid memory region,
and make a branch decision based on its value.
Impact was small (missed compression opportunity).
It only happens in -BD mode, with extDict-prefix overlapping matches.

The bt match search is supposed to work also in extDict mode.
In which case, the match ptr can point into Dict.
When the match was overlapping Dict<->Prefix,
match[matchLength] would end up outside of Dict, in an invalid memory area.
The correction ensures that in such a case,
match[matchLength] ends up at intended location, inside prefix.
---
 lib/lz4.c    | 14 +++++++++++++-
 lib/lz4opt.h |  5 +++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 93de1e5..87ec6ab 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -280,10 +280,22 @@ static const int LZ4_minLength = (MFLIMIT+1);
 
 
 /*-************************************
-*  Common Utils
+*  Error detection
 **************************************/
 #define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
+#  include 
+#  define DEBUGLOG(l, ...) {                          \
+                if (l<=LZ4_DEBUG) {                   \
+                    fprintf(stderr, __FILE__ ": ");   \
+                    fprintf(stderr, __VA_ARGS__);     \
+                    fprintf(stderr, " \n");           \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
+
 
 /*-************************************
 *  Common functions
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index b346eba..e9e54d8 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -122,6 +122,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
             matchLength = LZ4_count(ip, match, vLimit);
             if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
                 matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit);
+            if (matchIndex+matchLength >= dictLimit)
+                match = base + matchIndex;   /* to prepare for next usage of match[matchLength] */
         }
 
         if (matchLength > best_mlen) {
@@ -140,6 +142,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
         if (ip+matchLength >= iHighLimit)   /* equal : no way to know if inf or sup */
             break;   /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
 
+        DEBUGLOG(6, "ip   :%016llX", (U64)ip);
+        DEBUGLOG(6, "match:%016llX", (U64)match);
         if (*(ip+matchLength) < *(match+matchLength)) {
             *ptr0 = delta0;
             ptr0 = &DELTANEXTMAXD(matchIndex*2);
@@ -224,6 +228,7 @@ static int LZ4HC_compress_optimal (
     BYTE* const oend = op + maxOutputSize;
 
     /* init */
+    DEBUGLOG(5, "LZ4HC_compress_optimal");
     if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
     ctx->end += inputSize;
     ip++;
-- 
cgit v0.12


From 872f767fecea8bcd6f26fadcaf8d03ae98b5fdfc Mon Sep 17 00:00:00 2001
From: "FeRD (Frank Dana)" 
Date: Sat, 8 Jul 2017 07:43:51 -0400
Subject: Fix formatting of concatenation example

The "Concatenation of .lz4 files" section contains example commands
that are run together on one line, making them invalid. Wrap them
in a code block and clean up surrounding formatting.
---
 programs/lz4.1.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index 6f42904..c6b99bc 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -63,15 +63,11 @@ Default behaviors can be modified by opt-in commands, detailed below.
 It is possible to concatenate `.lz4` files as is.
 `lz4` will decompress such files as if they were a single `.lz4` file.
 For example:
+
     lz4 file1  > foo.lz4
     lz4 file2 >> foo.lz4
 
-then
-    lz4cat foo.lz4
-
-is equivalent to :
-    cat file1 file2
-
+Then `lz4cat foo.lz4` is equivalent to `cat file1 file2`.
 
 OPTIONS
 -------
-- 
cgit v0.12


From 63d5005681190c9db41321243d7d48d33f04b9be Mon Sep 17 00:00:00 2001
From: "FeRD (Frank Dana)" 
Date: Sat, 8 Jul 2017 11:56:55 -0400
Subject: Generate updated man page from Markdown source

---
 programs/lz4.1 | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/programs/lz4.1 b/programs/lz4.1
index 48c988a..39d78cd 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
 .
-.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands"
+.TH "LZ4" "1" "July 2017" "lz4 1.8.0" "User Commands"
 .
 .SH "NAME"
 \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -32,6 +32,9 @@ When writing scripts that need to decompress files, it is recommended to always
 \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2
 .
 .IP "\(bu" 4
+\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
+.
+.IP "\(bu" 4
 \fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them)
 .
 .IP "\(bu" 4
@@ -41,7 +44,7 @@ If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if
 If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
 .
 .IP "\(bu" 4
-As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\.
+As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
 .
 .IP "" 0
 .
@@ -49,7 +52,10 @@ As a consequence of previous rules, note the following example : \fBlz4 file | c
 Default behaviors can be modified by opt\-in commands, detailed below\.
 .
 .IP "\(bu" 4
-\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\.
+\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
+.
+.IP "\(bu" 4
+Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
 .
 .IP "\(bu" 4
 It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\.
@@ -60,13 +66,21 @@ Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\.
 .IP "" 0
 .
 .SS "Concatenation of \.lz4 files"
-It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4
+It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example:
 .
-.P
-then lz4cat foo\.lz4
+.IP "" 4
+.
+.nf
+
+lz4 file1  > foo\.lz4
+lz4 file2 >> foo\.lz4
+.
+.fi
+.
+.IP "" 0
 .
 .P
-is equivalent to : cat file1 file2
+Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\.
 .
 .SH "OPTIONS"
 .
@@ -181,6 +195,10 @@ Preserve source files (default behavior)
 \fB\-\-rm\fR
 Delete source files on successful compression or decompression
 .
+.TP
+\fB\-\-\fR
+Treat all subsequent arguments as files
+.
 .SS "Benchmark mode"
 .
 .TP
@@ -195,10 +213,6 @@ Benchmark multiple compression levels, from b# to e# (included)
 \fB\-i#\fR
 Minimum evaluation in seconds [1\-9] (default : 3)
 .
-.TP
-\fB\-r\fR
-Operate recursively on directories
-.
 .SH "BUGS"
 Report bugs at: https://github\.com/lz4/lz4/issues
 .
-- 
cgit v0.12


From 8aeaf6bb3a6e837e2443fca71b0fe20490b92578 Mon Sep 17 00:00:00 2001
From: Ido Rosen 
Date: Tue, 1 Aug 2017 00:48:58 -0400
Subject: Fix typos preventing installation of static lib.

---
 lib/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index c6fd7b8..de5c4ff 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -42,7 +42,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER  := $(shell echo $(LIBVER_SCRIPT))
 
-BUILD_STATIC:= yes
+BUILD_STATIC:=yes
 
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
 CFLAGS  ?= -O3
@@ -149,6 +149,7 @@ install: lib liblz4.pc
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
 	@$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
 	@$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-- 
cgit v0.12


From dba4d70a1dbaf205bc68c5c543c6ccef95fdeafe Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 6 Aug 2017 21:04:46 -0700
Subject: updated man page

---
 programs/Makefile |  3 +--
 programs/lz4.1    | 36 +++++++++++++++++++++++++-----------
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 8cf7d59..c484731 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -132,7 +132,6 @@ MANDIR  ?= $(PREFIX)/share/man/man1
 endif
 
 INSTALL_PROGRAM ?= $(INSTALL) -m 755
-INSTALL_SCRIPT  ?= $(INSTALL) -m 755
 INSTALL_MAN     ?= $(INSTALL) -m 644
 
 
@@ -144,7 +143,7 @@ install: lz4$(EXT) lz4c$(EXT)
 	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
 	@$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c
 	@echo Installing man pages
-	@$(INSTALL_MAN) -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
+	@$(INSTALL_MAN) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
diff --git a/programs/lz4.1 b/programs/lz4.1
index 48c988a..39d78cd 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
 .
-.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands"
+.TH "LZ4" "1" "July 2017" "lz4 1.8.0" "User Commands"
 .
 .SH "NAME"
 \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -32,6 +32,9 @@ When writing scripts that need to decompress files, it is recommended to always
 \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2
 .
 .IP "\(bu" 4
+\fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
+.
+.IP "\(bu" 4
 \fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them)
 .
 .IP "\(bu" 4
@@ -41,7 +44,7 @@ If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if
 If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\.
 .
 .IP "\(bu" 4
-As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\.
+As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
 .
 .IP "" 0
 .
@@ -49,7 +52,10 @@ As a consequence of previous rules, note the following example : \fBlz4 file | c
 Default behaviors can be modified by opt\-in commands, detailed below\.
 .
 .IP "\(bu" 4
-\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\.
+\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default (use \fB\-v\fR to enable them)\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main remaining difference being that source files are preserved by default\.
+.
+.IP "\(bu" 4
+Similarly, \fBlz4 \-m \-d\fR can decompress multiple \fB*\.lz4\fR files\.
 .
 .IP "\(bu" 4
 It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\.
@@ -60,13 +66,21 @@ Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\.
 .IP "" 0
 .
 .SS "Concatenation of \.lz4 files"
-It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4
+It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example:
 .
-.P
-then lz4cat foo\.lz4
+.IP "" 4
+.
+.nf
+
+lz4 file1  > foo\.lz4
+lz4 file2 >> foo\.lz4
+.
+.fi
+.
+.IP "" 0
 .
 .P
-is equivalent to : cat file1 file2
+Then \fBlz4cat foo\.lz4\fR is equivalent to \fBcat file1 file2\fR\.
 .
 .SH "OPTIONS"
 .
@@ -181,6 +195,10 @@ Preserve source files (default behavior)
 \fB\-\-rm\fR
 Delete source files on successful compression or decompression
 .
+.TP
+\fB\-\-\fR
+Treat all subsequent arguments as files
+.
 .SS "Benchmark mode"
 .
 .TP
@@ -195,10 +213,6 @@ Benchmark multiple compression levels, from b# to e# (included)
 \fB\-i#\fR
 Minimum evaluation in seconds [1\-9] (default : 3)
 .
-.TP
-\fB\-r\fR
-Operate recursively on directories
-.
 .SH "BUGS"
 Report bugs at: https://github\.com/lz4/lz4/issues
 .
-- 
cgit v0.12


From 76ef6d0ab0ea0937e8bb92c3a11a6445504d720f Mon Sep 17 00:00:00 2001
From: Alex Deymo 
Date: Fri, 28 Jul 2017 16:33:43 +0200
Subject: Allow to predefine FORCE_INLINE macro.

FORCE_INLINE macro is defined based on the compiler used. When using
gcc, it will include "__attribute__((always_inline))" forcing gcc to
always inline all the functions marked as FORCE_INLINE. However, this
can cause a performance degradation of about 15%.

This patch allows to set the FORCE_INLINE macro from the compiler
command line to either "static" or "static inline" giving allowing it
to inline functions as needed when performing optimizations.
---
 lib/lz4.c    | 25 ++++++++++++++++---------
 lib/xxhash.c | 27 ++++++++++++++++-----------
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 87ec6ab..62e4dc2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -96,20 +96,27 @@
 *  Compiler Options
 **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define FORCE_INLINE static __forceinline
 #  include 
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4293)        /* disable: C4293: too large shift (32-bits) */
-#else
-#  if defined(__GNUC__) || defined(__clang__)
-#    define FORCE_INLINE static inline __attribute__((always_inline))
-#  elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-#    define FORCE_INLINE static inline
-#  else
-#    define FORCE_INLINE static
-#  endif
 #endif  /* _MSC_VER */
 
+#ifndef FORCE_INLINE
+#  ifdef _MSC_VER    /* Visual Studio */
+#    define FORCE_INLINE static __forceinline
+#  else
+#    if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#      ifdef __GNUC__
+#        define FORCE_INLINE static inline __attribute__((always_inline))
+#      else
+#        define FORCE_INLINE static inline
+#      endif
+#    else
+#      define FORCE_INLINE static
+#    endif /* __STDC_VERSION__ */
+#  endif  /* _MSC_VER */
+#endif /* FORCE_INLINE */
+
 #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
 #  define expect(expr,value)    (__builtin_expect ((expr),(value)) )
 #else
diff --git a/lib/xxhash.c b/lib/xxhash.c
index e9ff2d4..a532358 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -113,19 +113,24 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
 ***************************************/
 #ifdef _MSC_VER    /* Visual Studio */
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
-#  define FORCE_INLINE static __forceinline
-#else
-#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
-#    ifdef __GNUC__
-#      define FORCE_INLINE static inline __attribute__((always_inline))
-#    else
-#      define FORCE_INLINE static inline
-#    endif
-#  else
-#    define FORCE_INLINE static
-#  endif /* __STDC_VERSION__ */
 #endif
 
+#ifndef FORCE_INLINE
+#  ifdef _MSC_VER    /* Visual Studio */
+#    define FORCE_INLINE static __forceinline
+#  else
+#    if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#      ifdef __GNUC__
+#        define FORCE_INLINE static inline __attribute__((always_inline))
+#      else
+#        define FORCE_INLINE static inline
+#      endif
+#    else
+#      define FORCE_INLINE static
+#    endif /* __STDC_VERSION__ */
+#  endif  /* _MSC_VER */
+#endif /* FORCE_INLINE */
+
 
 /* *************************************
 *  Basic Types
-- 
cgit v0.12


From e98a528576c1cc095dcdcf9ff1f4e3e8422a535d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 8 Aug 2017 11:59:57 -0700
Subject: updated Frame specification

Restored DictID field in Frame header
Bumped specification version to v1.6.0
---
 NEWS                    |  2 ++
 doc/lz4_Frame_format.md | 77 +++++++++++++++++++++++++++++++++----------------
 2 files changed, 54 insertions(+), 25 deletions(-)

diff --git a/NEWS b/NEWS
index f6df47c..d10c48f 100644
--- a/NEWS
+++ b/NEWS
@@ -5,8 +5,10 @@ API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
 API : added LZ4F_resetDecompressionContext()
 API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
+build : fix : static lib installation, by Ido Rosen
 build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
+doc : Updated LZ4 Frame format to v1.6.0, restoring Dictionary-ID field
 doc : lz4 api manual, by Przemyslaw Skibinski
 
 v1.7.5
diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index 5f8f479..bae2b06 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-1.5.1 (31/03/2015)
+1.6.0 (08/08/2017)
 
 
 Introduction
@@ -63,7 +63,7 @@ General Structure of LZ4 Frame format
 
 | MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum |
 |:-------:|:-------------:| ----- | ----- | ------- | ----------- |
-| 4 bytes |  3-11 bytes   |       |       | 4 bytes | 0-4 bytes   |
+| 4 bytes |  3-15 bytes   |       |       | 4 bytes | 0-4 bytes   |
 
 __Magic Number__
 
@@ -72,7 +72,7 @@ Value : 0x184D2204
 
 __Frame Descriptor__
 
-3 to 11 Bytes, to be detailed in the next part.
+3 to 15 Bytes, to be detailed in the next part.
 Most important part of the spec.
 
 __Data Blocks__
@@ -118,31 +118,31 @@ to decode all concatenated frames in their sequential order.
 Frame Descriptor
 ----------------
 
-| FLG     | BD      | (Content Size) | HC      |
-| ------- | ------- |:--------------:| ------- |
-| 1 byte  | 1 byte  |  0 - 8 bytes   | 1 byte  |
+| FLG     | BD      | (Content Size) | (Dictionary ID) | HC      |
+| ------- | ------- |:--------------:|:---------------:| ------- |
+| 1 byte  | 1 byte  |  0 - 8 bytes   |   0 - 4 bytes   | 1 byte  |
 
 The descriptor uses a minimum of 3 bytes,
-and up to 11 bytes depending on optional parameters.
+and up to 15 bytes depending on optional parameters.
 
 __FLG byte__
 
-|  BitNb  |   7-6   |    5    |     4     |   3     |     2     |    1-0   |
-| ------- | ------- | ------- | --------- | ------- | --------- | -------- |
-|FieldName| Version | B.Indep | B.Checksum| C.Size  | C.Checksum|*Reserved*|
+|  BitNb  |  7-6  |   5   |    4     |  3   |    2     |    1     |   0  |
+| ------- |-------|-------|----------|------|----------|----------|------|
+|FieldName|Version|B.Indep|B.Checksum|C.Size|C.Checksum|*Reserved*|DictID|
 
 
 __BD byte__
 
-|  BitNb  |     7    |     6-5-4    |  3-2-1-0 |
-| ------- | -------- | ------------ | -------- |
-|FieldName|*Reserved*| Block MaxSize|*Reserved*|
+|  BitNb  |     7    |     6-5-4     |  3-2-1-0 |
+| ------- | -------- | ------------- | -------- |
+|FieldName|*Reserved*| Block MaxSize |*Reserved*|
 
 In the tables, bit 7 is highest bit, while bit 0 is lowest.
 
 __Version Number__
 
-2-bits field, must be set to “01”.
+2-bits field, must be set to `01`.
 Any other value cannot be decoded by this version of the specification.
 Other version numbers will use different flag layouts.
 
@@ -154,7 +154,7 @@ If this flag is set to “0”, each block depends on previous ones
 In such case, it’s necessary to decode all blocks in sequence.
 
 Block dependency improves compression ratio, especially for small blocks.
-On the other hand, it makes direct jumps or multi-threaded decoding impossible.
+On the other hand, it makes random access or multi-threaded decoding impossible.
 
 __Block checksum flag__
 
@@ -172,13 +172,17 @@ Content Size usage is optional.
 
 __Content checksum flag__
 
-If this flag is set, a content checksum will be appended after the EndMark.
+If this flag is set, a 32-bits content checksum will be appended
+after the EndMark.
 
-Recommended value : “1” (content checksum is present)
+__Dictionary ID flag__
+
+If this flag is set, a 4-bytes Dict-ID field will be present,
+after the descriptor flags and the Content Size.
 
 __Block Maximum Size__
 
-This information is intended to help the decoder allocate memory.
+This information is useful to help the decoder allocate memory.
 Size here refers to the original (uncompressed) data size.
 Block Maximum Size is one value among the following table :
 
@@ -186,17 +190,17 @@ Block Maximum Size is one value among the following table :
 | --- | --- | --- | --- | ----- | ------ | ---- | ---- |
 | N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB |
 
-The decoder may refuse to allocate block sizes above a (system-specific) size.
+The decoder may refuse to allocate block sizes above any system-specific size.
 Unused values may be used in a future revision of the spec.
-A decoder conformant to the current version of the spec
-is only able to decode blocksizes defined in this spec.
+A decoder conformant with the current version of the spec
+is only able to decode block sizes defined in this spec.
 
 __Reserved bits__
 
 Value of reserved bits **must** be 0 (zero).
 Reserved bit might be used in a future version of the specification,
 typically enabling new optional features.
-If this happens, a decoder respecting the current version of the specification
+When this happens, a decoder respecting the current specification version
 shall not be able to decode such a frame.
 
 __Content Size__
@@ -208,10 +212,31 @@ Format is Little endian.
 This value is informational, typically for display or memory allocation.
 It can be skipped by a decoder, or used to validate content correctness.
 
+__Dictionary ID__
+
+Dict-ID is only present if the associated flag is set.
+It's an unsigned 32-bits value, stored using little-endian convention.
+A dictionary is useful to compress short input sequences.
+The compressor can take advantage of the dictionary context
+to encode the input in a more compact manner.
+It works as a kind of “known prefix” which is used by
+both the compressor and the decompressor to “warm-up” reference tables.
+
+The decompressor can use Dict-ID identifier to determine
+which dictionary must be used to correctly decode data.
+The compressor and the decompressor must use exactly the same dictionary.
+It's presumed that the 32-bits dictID uniquely identifies a dictionary.
+
+Within a single frame, a single dictionary can be defined.
+When the frame descriptor defines independent blocks,
+each block will be initialized with the same dictionary.
+If the frame descriptor defines linked blocks,
+the dictionary will only be used once, at the beginning of the frame.
+
 __Header Checksum__
 
 One-byte checksum of combined descriptor fields, including optional ones.
-The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF `
+The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF `
 using zero as a seed,
 and the full Frame Descriptor as an input
 (including optional fields when they are present).
@@ -347,7 +372,7 @@ wether it is a file or a stream.
 
 Alternatively, if the frame is followed by a valid Frame Magic Number,
 it is considered completed.
-It makes legacy frames compatible with frame concatenation.
+This policy makes it possible to concatenate legacy frames.
 
 Any other value will be interpreted as a block size,
 and trigger an error if it does not fit within acceptable range.
@@ -356,7 +381,9 @@ and trigger an error if it does not fit within acceptable range.
 Version changes
 ---------------
 
-1.5.1 : changed format to MarkDown compatible
+1.6.0 : restored Dictionary ID field in Frame header
+
+1.5.1 : changed document format to MarkDown
 
 1.5 : removed Dictionary ID from specification
 
-- 
cgit v0.12


From a82dadfbae74916aecdd10121cc0177fd8162b90 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 8 Aug 2017 17:43:11 -0700
Subject: added dictID inside LZ4F_frameInfo_t

Compressor can set dictID on LZ4F_compressBegin()
Decompressor can retrieve it using LZ4F_getFrameInfo()
---
 doc/lz4_Frame_format.md |   3 +-
 lib/lz4frame.c          |  41 ++++++++----
 lib/lz4frame.h          |   5 +-
 tests/frametest.c       | 161 +++++++++++++++++++++++-------------------------
 4 files changed, 108 insertions(+), 102 deletions(-)

diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md
index bae2b06..77454b2 100644
--- a/doc/lz4_Frame_format.md
+++ b/doc/lz4_Frame_format.md
@@ -237,8 +237,7 @@ __Header Checksum__
 
 One-byte checksum of combined descriptor fields, including optional ones.
 The value is the second byte of `xxh32()` : ` (xxh32()>>8) & 0xFF `
-using zero as a seed,
-and the full Frame Descriptor as an input
+using zero as a seed, and the full Frame Descriptor as an input
 (including optional fields when they are present).
 A wrong checksum indicates an error in the descriptor.
 Header checksum is informational and can be skipped.
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 994cc8b..c2d6d5c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
 #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
 
 static const size_t minFHSize = 7;
-static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX;   /* 15 */
+static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX;   /* 19 */
 static const size_t BHSize = 4;
 
 
@@ -291,7 +291,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
 size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
     LZ4F_preferences_t prefs;
-    size_t const headerSize = maxFHSize;      /* max header size, including magic number and frame content size */
+    size_t const headerSize = maxFHSize;      /* max header size, including optional fields */
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
     else memset(&prefs, 0, sizeof(prefs));
@@ -470,9 +470,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
 
     /* FLG Byte */
     *dstPtr++ = (BYTE)(((1 & _2BITS) << 6)    /* Version('01') */
-        + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)    /* Block mode */
+        + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)  /* Block mode */
+        + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)     /* Frame content size */
         + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)   /* Frame checksum */
-        + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3));   /* Frame content size */
+        +  (cctxPtr->prefs.frameInfo.dictID > 0) );   /* Dictionary ID */
     /* BD Byte */
     *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
     /* Optional Frame content size field */
@@ -481,6 +482,11 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
         dstPtr += 8;
         cctxPtr->totalInSize = 0;
     }
+    /* Optional dictionary ID field */
+    if (cctxPtr->prefs.frameInfo.dictID) {
+        LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
+        dstPtr += 4;
+    }
     /* CRC Byte */
     *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
     dstPtr++;
@@ -817,12 +823,14 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
     if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
 
     /* control magic number */
-    if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
+    if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
+        return err0r(LZ4F_ERROR_frameType_unknown);
 
     /* Frame Header Size */
     {   BYTE const FLG = ((const BYTE*)src)[4];
         U32 const contentSizeFlag = (FLG>>3) & _1BIT;
-        return contentSizeFlag ? maxFHSize : minFHSize;
+        U32 const dictIDFlag = FLG & _1BIT;
+        return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
     }
 }
 
@@ -837,7 +845,7 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
  */
 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
 {
-    unsigned blockMode, contentSizeFlag, contentChecksumFlag, blockSizeID;
+    unsigned blockMode, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
     size_t frameHeaderSize;
     const BYTE* srcPtr = (const BYTE*)src;
 
@@ -860,7 +868,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     }
 
     /* control magic number */
-    if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown);
+    if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
+        return err0r(LZ4F_ERROR_frameType_unknown);
     dctxPtr->frameInfo.frameType = LZ4F_frame;
 
     /* Flags */
@@ -870,14 +879,15 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
         blockMode = (FLG>>5) & _1BIT;
         contentSizeFlag = (FLG>>3) & _1BIT;
         contentChecksumFlag = (FLG>>2) & _1BIT;
+        dictIDFlag = FLG & _1BIT;
         /* validate */
-        if (((FLG>>0)&_2BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+        if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
         if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong);        /* Version Number, only supported value */
         if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
     }
 
     /* Frame Header Size */
-    frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize;
+    frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
 
     if (srcSize < frameHeaderSize) {
         /* not enough input to fully decode frame header */
@@ -898,8 +908,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     }
 
     /* check header */
-    { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
-      if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); }
+    {   BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
+        if (HC != srcPtr[frameHeaderSize-1])
+            return err0r(LZ4F_ERROR_headerChecksum_invalid);
+    }
 
     /* save */
     dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
@@ -907,7 +919,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
     dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
     if (contentSizeFlag)
-        dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+        dctxPtr->frameRemainingSize =
+            dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+    if (dictIDFlag)
+        dctxPtr->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
 
     dctxPtr->dStage = dstage_init;
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index dd2be58..0efe220 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -162,7 +162,8 @@ typedef struct {
   LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
   LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
   unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
-  unsigned               reserved[2];           /* must be zero for forward compatibility */
+  unsigned               dictID;                /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               reserved[1];           /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
@@ -228,7 +229,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 /*----    Compression    ----*/
 
-#define LZ4F_HEADER_SIZE_MAX 15
+#define LZ4F_HEADER_SIZE_MAX 19
 /*! LZ4F_compressBegin() :
  * will write the frame header into dstBuffer.
  * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
diff --git a/tests/frametest.c b/tests/frametest.c
index 0dadf9f..02d7e5b 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -164,6 +164,9 @@ static unsigned FUZ_highbit(U32 v32)
 /*-*******************************************************
 *  Tests
 *********************************************************/
+#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) goto _output_error
+#define CHECK(f)   { LZ4F_errorCode_t const CHECK_V(err_ , f); }
+
 int basicTests(U32 seed, double compressibility)
 {
 #define COMPRESSIBLE_NOISE_LENGTH (2 MB)
@@ -197,24 +200,20 @@ int basicTests(U32 seed, double compressibility)
     /* Special case : null-content frame */
     testSize = 0;
     DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
     DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
 
     DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
-    { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
-      if (LZ4F_isError(errorCode)) goto _output_error; }
+    CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
 
     DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
     {   size_t avail_in = cSize;
         LZ4F_frameInfo_t frame_info;
-        LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
     }
 
     DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
-    { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx);
-      if (LZ4F_isError(errorCode)) goto _output_error; }
+    CHECK( LZ4F_freeDecompressionContext(dCtx) );
     dCtx = NULL;
 
     /* test one-pass frame compression */
@@ -224,26 +223,22 @@ int basicTests(U32 seed, double compressibility)
     {   LZ4F_preferences_t fastCompressPrefs;
         memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
         fastCompressPrefs.compressionLevel = -3;
-        cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs);
-        if (LZ4F_isError(cSize)) goto _output_error;
+        CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
         DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
     }
 
     DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
     DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
 
     DISPLAYLEVEL(3, "Decompression test : \n");
     {   size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
         size_t compressedBufferSize = cSize;
 
-        LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
 
         DISPLAYLEVEL(3, "Single Pass decompression : ");
-        { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
-          if (LZ4F_isError(decompressError)) goto _output_error; }
+        CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
         { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
           if (crcDest != crcOrig) goto _output_error; }
         DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
@@ -257,8 +252,7 @@ int basicTests(U32 seed, double compressibility)
             BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
             size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
             DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
-            decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
-            if (LZ4F_isError(decResult)) goto _output_error;
+            CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
             if (decResult != missingBytes) {
                 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
                 goto _output_error;
@@ -282,9 +276,9 @@ int basicTests(U32 seed, double compressibility)
             const BYTE* ip = (BYTE*)compressedBuffer;
 
             DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
-            errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
-            if (LZ4F_isError(errorCode)) goto _output_error;
-            DISPLAYLEVEL(3, " %u  \n", (unsigned)errorCode);
+            CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
+            //DISPLAYLEVEL(3, " %u  \n", (unsigned)errorCode);
+            DISPLAYLEVEL(3, " OK  \n");
 
             DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
             {   size_t nullSize = 0;
@@ -309,8 +303,7 @@ int basicTests(U32 seed, double compressibility)
 
             DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
             iSize = 15 - iSize;
-            errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
-            if (LZ4F_isError(errorCode)) goto _output_error;
+            CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
             DISPLAYLEVEL(3, " correctly decoded \n");
             ip += iSize;
         }
@@ -342,8 +335,7 @@ int basicTests(U32 seed, double compressibility)
             while (ip < iend) {
                 size_t oSize = oend-op;
                 size_t iSize = 1;
-                errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
-                if (LZ4F_isError(errorCode)) goto _output_error;
+                CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
                 op += oSize;
                 ip += iSize;
             }
@@ -356,21 +348,18 @@ int basicTests(U32 seed, double compressibility)
     DISPLAYLEVEL(3, "Using 64 KB block : ");
     prefs.frameInfo.blockSizeID = LZ4F_max64KB;
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "without checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "Using 256 KB block : ");
     prefs.frameInfo.blockSizeID = LZ4F_max256KB;
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "Decompression test : \n");
@@ -388,8 +377,7 @@ int basicTests(U32 seed, double compressibility)
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
-            { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
-              if (LZ4F_isError(decompressError)) goto _output_error; }
+            CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
         }
@@ -399,28 +387,24 @@ int basicTests(U32 seed, double compressibility)
             DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
          }
 
-        { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx);
-          if (LZ4F_isError(freeError)) goto _output_error; }
+        CHECK( LZ4F_freeDecompressionContext(dCtx) );
         dCtx = NULL;
     }
 
     DISPLAYLEVEL(3, "without checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "Using 1 MB block : ");
     prefs.frameInfo.blockSizeID = LZ4F_max1MB;
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "without checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
-    cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
-    if (LZ4F_isError(cSize)) goto _output_error;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
     DISPLAYLEVEL(3, "Using 4 MB block : ");
@@ -428,8 +412,7 @@ int basicTests(U32 seed, double compressibility)
     prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
     {   size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
         DISPLAYLEVEL(4, "dstCapacity = %u  ; ", (U32)dstCapacity)
-        cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
-        if (LZ4F_isError(cSize)) goto _output_error;
+        CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
         DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
     }
 
@@ -437,61 +420,72 @@ int basicTests(U32 seed, double compressibility)
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
     {   size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
         DISPLAYLEVEL(4, "dstCapacity = %u  ; ", (U32)dstCapacity)
-        cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
-        if (LZ4F_isError(cSize)) goto _output_error;
+        CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
         DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
     }
 
-    {   size_t errorCode;
+    /* frame content size tests */
+    {   size_t cErr;
         BYTE* const ostart = (BYTE*)compressedBuffer;
         BYTE* op = ostart;
-        errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
 
         DISPLAYLEVEL(3, "compress without frameSize : ");
         memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
-        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+        op += cErr;
+        CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+        op += cErr;
+        CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
         DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
 
         DISPLAYLEVEL(3, "compress with frameSize : ");
         prefs.frameInfo.contentSize = testSize;
         op = ostart;
-        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+        op += cErr;
+        CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+        op += cErr;
+        CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
         DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
 
         DISPLAYLEVEL(3, "compress with wrong frameSize : ");
         prefs.frameInfo.contentSize = testSize+1;
         op = ostart;
-        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
-        if (LZ4F_isError(errorCode)) goto _output_error;
-        op += errorCode;
-        errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
-        if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
-        else
-            goto _output_error;
-
-        errorCode = LZ4F_freeCompressionContext(cctx);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+        op += cErr;
+        CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
+        op += cErr;
+        cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
+        if (!LZ4F_isError(cErr)) goto _output_error;
+        DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
+
+        CHECK( LZ4F_freeCompressionContext(cctx) );
         cctx = NULL;
     }
 
+    /* dictID tests */
+    {   size_t cErr;
+        U32 const dictID = 0x99;
+        CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
+
+        DISPLAYLEVEL(3, "insert a dictID : ");
+        memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
+        prefs.frameInfo.dictID = dictID;
+        CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
+        DISPLAYLEVEL(3, "created frame header of size %i bytes  \n", (int)cErr);
+
+        DISPLAYLEVEL(3, "read a dictID : ");
+        CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
+        memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
+        CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
+        if (prefs.frameInfo.dictID != dictID) goto _output_error;
+        DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
+
+        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
+        CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
+    }
+
     DISPLAYLEVEL(3, "Skippable frame test : \n");
     {   size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
         unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
@@ -500,8 +494,7 @@ int basicTests(U32 seed, double compressibility)
         BYTE* ip = (BYTE*)compressedBuffer;
         BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
 
-        LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
-        if (LZ4F_isError(errorCode)) goto _output_error;
+        CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
 
         /* generate skippable frame */
         FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
@@ -513,8 +506,7 @@ int basicTests(U32 seed, double compressibility)
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
-            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
-            if (LZ4F_isError(errorCode)) goto _output_error;
+            CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
         }
@@ -533,8 +525,7 @@ int basicTests(U32 seed, double compressibility)
             size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip;
-            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
-            if (LZ4F_isError(errorCode)) goto _output_error;
+            CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
         }
@@ -550,8 +541,7 @@ int basicTests(U32 seed, double compressibility)
             size_t iSize = 10;
             size_t oSize = 10;
             if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
-            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
-            if (LZ4F_isError(errorCode)) goto _output_error;
+            CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
             op += oSize;
             ip += iSize;
         }
@@ -603,6 +593,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
     size_t result;
     clock_t const startClock = clock();
     clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
+#   undef CHECK
 #   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                             DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
 
-- 
cgit v0.12


From 1d1737aaf28c60d3cf6950b545f6a844afe422c0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 9 Aug 2017 12:29:38 -0700
Subject: fixed frameCompress example

---
 doc/lz4frame_manual.html |  35 +--
 examples/frameCompress.c | 537 ++++++++++++++++++++++++-----------------------
 2 files changed, 288 insertions(+), 284 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 87750a1..e6873c1 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -74,7 +74,8 @@
   LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
   LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
   unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
-  unsigned               reserved[2];           /* must be zero for forward compatibility */
+  unsigned               dictID;                /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               reserved[1];           /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 makes it possible to supply detailed frame parameters to the stream interface.
  It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -186,15 +187,15 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
Decompression functions

 
 
typedef struct {
-  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
   unsigned reserved[3];
 } LZ4F_decompressOptions_t;
 


 
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-
 Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
- The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
- The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+
 Create an LZ4F_dctx object, to track all decompression operations.
+ The version provided MUST be LZ4F_VERSION.
+ The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  The result is an errorCode, which can be tested using LZ4F_isError().
  dctx memory can be released using LZ4F_freeDecompressionContext();
  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
@@ -208,20 +209,22 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
 This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- Its usage is optional. Extracted information can be useful for allocation purposes, typically.
+ Its usage is optional.
+ Extracted information can typically be useful for allocation purposes.
  This function works in 2 situations :
-   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+   - At the beginning of a new frame, in which case
+     it will decode information from `srcBuffer`, starting the decoding process.
      Input size must be large enough to successfully decode the entire frame header.
      Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
      It's allowed to provide more input data than this minimum.
    - After decoding has been started.
      In which case, no input is read, frame parameters are extracted from dctx.
-     If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
+   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
  Decompression must resume from (srcBuffer + *srcSizePtr).
  @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
            or an error code which can be tested using LZ4F_isError()
- note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  
 


@@ -230,18 +233,18 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                    void* dstBuffer, size_t* dstSizePtr,
                                    const void* srcBuffer, size_t* srcSizePtr,
                                    const LZ4F_decompressOptions_t* dOptPtr);
-
 Call this function repetitively to regenerate data compressed within `srcBuffer`.
+
 Call this function repetitively to regenerate compressed data from `srcBuffer`.
  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
 
- The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
 
- The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
+ The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ Number of bytes consumed can be < number of bytes provided.
  It typically happens when dstBuffer is not large enough to contain all decoded data.
- Remaining data will have to be presented again in a subsequent invocation.
+ Unconsumed source data must be presented again in subsequent invocations.
 
  `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- `dstBuffer` can be changed at will between each consecutive function invocation.
+ `dstBuffer` itself can be changed at will between each consecutive function invocation.
 
  @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
@@ -259,7 +262,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
  In case of an error, the context is left in "undefined" state.
   In which case, it's necessary to reset it, before re-using it.
   This method can also be used to abruptly stop an unfinished decompression,
-  and start a new with the same context. 
+  and start a new one using the same context. 
 


 
 
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 75f1576..0a42fe3 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -13,299 +13,300 @@
 #define LZ4_FOOTER_SIZE 4
 
 static const LZ4F_preferences_t lz4_preferences = {
-	{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0, { 0, 0 } },
-	0,   /* compression level */
-	0,   /* autoflush */
-	{ 0, 0, 0, 0 },  /* reserved, must be set to 0 */
+    { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
+      0 /* content size unknown */, 0 /* no dictID */ , { 0 } /* reserved */ },
+    0,   /* compression level */
+    0,   /* autoflush */
+    { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
-	LZ4F_errorCode_t r;
-	LZ4F_compressionContext_t ctx;
-	char *src, *buf = NULL;
-	size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
-
-	r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
-	if (LZ4F_isError(r)) {
-		printf("Failed to create context: error %zu\n", r);
-		return 1;
-	}
-	r = 1;
-
-	src = malloc(BUF_SIZE);
-	if (!src) {
-		printf("Not enough memory\n");
-		goto cleanup;
-	}
-
-	frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
-	size =  frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
-	buf = malloc(size);
-	if (!buf) {
-		printf("Not enough memory\n");
-		goto cleanup;
-	}
-
-	n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
-	if (LZ4F_isError(n)) {
-		printf("Failed to start compression: error %zu\n", n);
-		goto cleanup;
-	}
-
-	printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
-
-	for (;;) {
-		k = fread(src, 1, BUF_SIZE, in);
-		if (k == 0)
-			break;
-		count_in += k;
-
-		n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
-		if (LZ4F_isError(n)) {
-			printf("Compression failed: error %zu\n", n);
-			goto cleanup;
-		}
-
-		offset += n;
-		count_out += n;
-		if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
-			printf("Writing %zu bytes\n", offset);
-
-			k = fwrite(buf, 1, offset, out);
-			if (k < offset) {
-				if (ferror(out))
-					printf("Write failed\n");
-				else
-					printf("Short write\n");
-				goto cleanup;
-			}
-
-			offset = 0;
-		}
-	}
-
-	n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
-	if (LZ4F_isError(n)) {
-		printf("Failed to end compression: error %zu\n", n);
-		goto cleanup;
-	}
-
-	offset += n;
-	count_out += n;
-	printf("Writing %zu bytes\n", offset);
-
-	k = fwrite(buf, 1, offset, out);
-	if (k < offset) {
-		if (ferror(out))
-			printf("Write failed\n");
-		else
-			printf("Short write\n");
-		goto cleanup;
-	}
-
-	*size_in = count_in;
-	*size_out = count_out;
-	r = 0;
+    LZ4F_errorCode_t r;
+    LZ4F_compressionContext_t ctx;
+    char *src, *buf = NULL;
+    size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
+
+    r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+    if (LZ4F_isError(r)) {
+        printf("Failed to create context: error %zu\n", r);
+        return 1;
+    }
+    r = 1;  /* function result; 1 == error, by default (early exit) */
+
+    src = malloc(BUF_SIZE);
+    if (!src) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
+    size =  frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
+    buf = malloc(size);
+    if (!buf) {
+        printf("Not enough memory\n");
+        goto cleanup;
+    }
+
+    n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+    if (LZ4F_isError(n)) {
+        printf("Failed to start compression: error %zu\n", n);
+        goto cleanup;
+    }
+
+    printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
+
+    for (;;) {
+        k = fread(src, 1, BUF_SIZE, in);
+        if (k == 0)
+            break;
+        count_in += k;
+
+        n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
+        if (LZ4F_isError(n)) {
+            printf("Compression failed: error %zu\n", n);
+            goto cleanup;
+        }
+
+        offset += n;
+        count_out += n;
+        if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
+            printf("Writing %zu bytes\n", offset);
+
+            k = fwrite(buf, 1, offset, out);
+            if (k < offset) {
+                if (ferror(out))
+                    printf("Write failed\n");
+                else
+                    printf("Short write\n");
+                goto cleanup;
+            }
+
+            offset = 0;
+        }
+    }
+
+    n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
+    if (LZ4F_isError(n)) {
+        printf("Failed to end compression: error %zu\n", n);
+        goto cleanup;
+    }
+
+    offset += n;
+    count_out += n;
+    printf("Writing %zu bytes\n", offset);
+
+    k = fwrite(buf, 1, offset, out);
+    if (k < offset) {
+        if (ferror(out))
+            printf("Write failed\n");
+        else
+            printf("Short write\n");
+        goto cleanup;
+    }
+
+    *size_in = count_in;
+    *size_out = count_out;
+    r = 0;
  cleanup:
-	if (ctx)
-		LZ4F_freeCompressionContext(ctx);
-	free(src);
-	free(buf);
-	return r;
+    if (ctx)
+        LZ4F_freeCompressionContext(ctx);
+    free(src);
+    free(buf);
+    return r;
 }
 
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
-	switch (info->blockSizeID) {
+    switch (info->blockSizeID) {
         case LZ4F_default:
-		case LZ4F_max64KB:  return 1 << 16;
-		case LZ4F_max256KB: return 1 << 18;
-		case LZ4F_max1MB:   return 1 << 20;
-		case LZ4F_max4MB:   return 1 << 22;
-		default:
-			printf("Impossible unless more block sizes are allowed\n");
-			exit(1);
-	}
+        case LZ4F_max64KB:  return 1 << 16;
+        case LZ4F_max256KB: return 1 << 18;
+        case LZ4F_max1MB:   return 1 << 20;
+        case LZ4F_max4MB:   return 1 << 22;
+        default:
+            printf("Impossible unless more block sizes are allowed\n");
+            exit(1);
+    }
 }
 
 static size_t decompress_file(FILE *in, FILE *out) {
-	void* const src = malloc(BUF_SIZE);
-	void* dst = NULL;
-	size_t dstCapacity = 0;
-	LZ4F_dctx *dctx = NULL;
-	size_t ret;
+    void* const src = malloc(BUF_SIZE);
+    void* dst = NULL;
+    size_t dstCapacity = 0;
+    LZ4F_dctx *dctx = NULL;
+    size_t ret;
 
-	/* Initialization */
+    /* Initialization */
     if (!src) { perror("decompress_file(src)"); goto cleanup; }
-	ret = LZ4F_createDecompressionContext(&dctx, 100);
-	if (LZ4F_isError(ret)) {
-		printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
-		goto cleanup;
-	}
-
-	/* Decompression */
-	ret = 1;
-	while (ret != 0) {
-		/* Load more input */
-		size_t srcSize = fread(src, 1, BUF_SIZE, in);
-		void* srcPtr = src;
-		void* srcEnd = srcPtr + srcSize;
-		if (srcSize == 0 || ferror(in)) {
-			printf("Decompress: not enough input or error reading file\n");
-			goto cleanup;
-		}
-		/* Allocate destination buffer if it isn't already */
-		if (!dst) {
-			LZ4F_frameInfo_t info;
-			ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
-			if (LZ4F_isError(ret)) {
-				printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
-				goto cleanup;
-			}
-			/* Allocating enough space for an entire block isn't necessary for
-			 * correctness, but it allows some memcpy's to be elided.
-			 */
-			dstCapacity = get_block_size(&info);
-			dst = malloc(dstCapacity);
+    ret = LZ4F_createDecompressionContext(&dctx, 100);
+    if (LZ4F_isError(ret)) {
+        printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
+        goto cleanup;
+    }
+
+    /* Decompression */
+    ret = 1;
+    while (ret != 0) {
+        /* Load more input */
+        size_t srcSize = fread(src, 1, BUF_SIZE, in);
+        void* srcPtr = src;
+        void* srcEnd = srcPtr + srcSize;
+        if (srcSize == 0 || ferror(in)) {
+            printf("Decompress: not enough input or error reading file\n");
+            goto cleanup;
+        }
+        /* Allocate destination buffer if it isn't already */
+        if (!dst) {
+            LZ4F_frameInfo_t info;
+            ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize);
+            if (LZ4F_isError(ret)) {
+                printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret));
+                goto cleanup;
+            }
+            /* Allocating enough space for an entire block isn't necessary for
+             * correctness, but it allows some memcpy's to be elided.
+             */
+            dstCapacity = get_block_size(&info);
+            dst = malloc(dstCapacity);
             if (!dst) { perror("decompress_file(dst)"); goto cleanup; }
-			srcPtr += srcSize;
-			srcSize = srcEnd - srcPtr;
-		}
-		/* Decompress:
-		 * Continue while there is more input to read and the frame isn't over.
-		 * If srcPtr == srcEnd then we know that there is no more output left in the
-		 * internal buffer left to flush.
-		 */
-		while (srcPtr != srcEnd && ret != 0) {
-			/* INVARIANT: Any data left in dst has already been written */
-			size_t dstSize = dstCapacity;
-			ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
-			if (LZ4F_isError(ret)) {
-				printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
-				goto cleanup;
-			}
-			/* Flush output */
-			if (dstSize != 0){
-				size_t written = fwrite(dst, 1, dstSize, out);
-				printf("Writing %zu bytes\n", dstSize);
-				if (written != dstSize) {
-					printf("Decompress: Failed to write to file\n");
-					goto cleanup;
-				}
-			}
-			/* Update input */
-			srcPtr += srcSize;
-			srcSize = srcEnd - srcPtr;
-		}
-	}
-	/* Check that there isn't trailing input data after the frame.
-	 * It is valid to have multiple frames in the same file, but this example
-	 * doesn't support it.
-	 */
-	ret = fread(src, 1, 1, in);
-	if (ret != 0 || !feof(in)) {
-		printf("Decompress: Trailing data left in file after frame\n");
-		goto cleanup;
-	}
+            srcPtr += srcSize;
+            srcSize = srcEnd - srcPtr;
+        }
+        /* Decompress:
+         * Continue while there is more input to read and the frame isn't over.
+         * If srcPtr == srcEnd then we know that there is no more output left in the
+         * internal buffer left to flush.
+         */
+        while (srcPtr != srcEnd && ret != 0) {
+            /* INVARIANT: Any data left in dst has already been written */
+            size_t dstSize = dstCapacity;
+            ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
+            if (LZ4F_isError(ret)) {
+                printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
+                goto cleanup;
+            }
+            /* Flush output */
+            if (dstSize != 0){
+                size_t written = fwrite(dst, 1, dstSize, out);
+                printf("Writing %zu bytes\n", dstSize);
+                if (written != dstSize) {
+                    printf("Decompress: Failed to write to file\n");
+                    goto cleanup;
+                }
+            }
+            /* Update input */
+            srcPtr += srcSize;
+            srcSize = srcEnd - srcPtr;
+        }
+    }
+    /* Check that there isn't trailing input data after the frame.
+     * It is valid to have multiple frames in the same file, but this example
+     * doesn't support it.
+     */
+    ret = fread(src, 1, 1, in);
+    if (ret != 0 || !feof(in)) {
+        printf("Decompress: Trailing data left in file after frame\n");
+        goto cleanup;
+    }
 
 cleanup:
-	free(src);
-	free(dst);
-	return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
+    free(src);
+    free(dst);
+    return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
 }
 
 int compare(FILE* fp0, FILE* fp1)
 {
-	int result = 0;
+    int result = 0;
 
-	while(0 == result) {
-		char b0[1024];
-		char b1[1024];
-		const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
-		const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
+    while(0 == result) {
+        char b0[1024];
+        char b1[1024];
+        const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
+        const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
 
-		result = (int) r0 - (int) r1;
+        result = (int) r0 - (int) r1;
 
-		if (0 == r0 || 0 == r1) {
-			break;
-		}
-		if (0 == result) {
-			result = memcmp(b0, b1, r0);
-		}
-	}
+        if (0 == r0 || 0 == r1) {
+            break;
+        }
+        if (0 == result) {
+            result = memcmp(b0, b1, r0);
+        }
+    }
 
-	return result;
+    return result;
 }
 
 int main(int argc, const char **argv) {
-	char inpFilename[256] = { 0 };
-	char lz4Filename[256] = { 0 };
-	char decFilename[256] = { 0 };
-
-	if(argc < 2) {
-		printf("Please specify input filename\n");
-		return 0;
-	}
-
-	snprintf(inpFilename, 256, "%s", argv[1]);
-	snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
-	snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
-
-	printf("inp = [%s]\n", inpFilename);
-	printf("lz4 = [%s]\n", lz4Filename);
-	printf("dec = [%s]\n", decFilename);
-
-	/* compress */
-	{   FILE* const inpFp = fopen(inpFilename, "rb");
-		FILE* const outFp = fopen(lz4Filename, "wb");
-		size_t sizeIn = 0;
-		size_t sizeOut = 0;
-		size_t ret;
-
-		printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-		ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
-		if (ret) {
-			printf("compress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("%s: %zu → %zu bytes, %.1f%%\n",
-			inpFilename, sizeIn, sizeOut,
-			(double)sizeOut / sizeIn * 100);
-		printf("compress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* decompress */
-	{   FILE* const inpFp = fopen(lz4Filename, "rb");
-		FILE* const outFp = fopen(decFilename, "wb");
-		size_t ret;
-
-		printf("decompress : %s -> %s\n", lz4Filename, decFilename);
-		ret = decompress_file(inpFp, outFp);
-		if (ret) {
-			printf("decompress : failed with code %zu\n", ret);
-			return ret;
-		}
-		printf("decompress : done\n");
-
-		fclose(outFp);
-		fclose(inpFp);
-	}
-
-	/* verify */
-	{   FILE* const inpFp = fopen(inpFilename, "rb");
-		FILE* const decFp = fopen(decFilename, "rb");
-
-		printf("verify : %s <-> %s\n", inpFilename, decFilename);
-		const int cmp = compare(inpFp, decFp);
-		if(0 == cmp) {
-			printf("verify : OK\n");
-		} else {
-			printf("verify : NG\n");
-		}
-
-		fclose(decFp);
-		fclose(inpFp);
-	}
+    char inpFilename[256] = { 0 };
+    char lz4Filename[256] = { 0 };
+    char decFilename[256] = { 0 };
+
+    if(argc < 2) {
+        printf("Please specify input filename\n");
+        return 0;
+    }
+
+    snprintf(inpFilename, 256, "%s", argv[1]);
+    snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
+    snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
+
+    printf("inp = [%s]\n", inpFilename);
+    printf("lz4 = [%s]\n", lz4Filename);
+    printf("dec = [%s]\n", decFilename);
+
+    /* compress */
+    {   FILE* const inpFp = fopen(inpFilename, "rb");
+        FILE* const outFp = fopen(lz4Filename, "wb");
+        size_t sizeIn = 0;
+        size_t sizeOut = 0;
+        size_t ret;
+
+        printf("compress : %s -> %s\n", inpFilename, lz4Filename);
+        ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
+        if (ret) {
+            printf("compress : failed with code %zu\n", ret);
+            return ret;
+        }
+        printf("%s: %zu → %zu bytes, %.1f%%\n",
+            inpFilename, sizeIn, sizeOut,
+            (double)sizeOut / sizeIn * 100);
+        printf("compress : done\n");
+
+        fclose(outFp);
+        fclose(inpFp);
+    }
+
+    /* decompress */
+    {   FILE* const inpFp = fopen(lz4Filename, "rb");
+        FILE* const outFp = fopen(decFilename, "wb");
+        size_t ret;
+
+        printf("decompress : %s -> %s\n", lz4Filename, decFilename);
+        ret = decompress_file(inpFp, outFp);
+        if (ret) {
+            printf("decompress : failed with code %zu\n", ret);
+            return ret;
+        }
+        printf("decompress : done\n");
+
+        fclose(outFp);
+        fclose(inpFp);
+    }
+
+    /* verify */
+    {   FILE* const inpFp = fopen(inpFilename, "rb");
+        FILE* const decFp = fopen(decFilename, "rb");
+
+        printf("verify : %s <-> %s\n", inpFilename, decFilename);
+        const int cmp = compare(inpFp, decFp);
+        if(0 == cmp) {
+            printf("verify : OK\n");
+        } else {
+            printf("verify : NG\n");
+        }
+
+        fclose(decFp);
+        fclose(inpFp);
+    }
 }
-- 
cgit v0.12


From 31f2cdf4d214edcd9d4496058b0f9a4c4f8c0fad Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 9 Aug 2017 16:51:19 -0700
Subject: implemented dictionary compression in lz4frame

note : only compression API is implemented and tested
still to do : decompression API
---
 doc/lz4_manual.html      |   2 +-
 doc/lz4frame_manual.html |  36 ++++----
 lib/lz4.c                |   7 +-
 lib/lz4.h                |   2 +-
 lib/lz4frame.c           | 227 +++++++++++++++++++++++++++++++++--------------
 lib/lz4frame.h           |  39 ++++----
 lib/lz4frame_static.h    |  44 +++++++++
 lib/lz4hc.c              |  12 ++-
 tests/frametest.c        |  28 +++++-
 9 files changed, 285 insertions(+), 112 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index fecd8cd..60b41a8 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -277,7 +277,7 @@ union LZ4_stream_u {
  init this structure before first use.
  note : only use in association with static linking !
         this definition is not API/ABI safe,
-        and may change in a future version !
+        it may change in a future version !
  
 


 
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index e6873c1..dc8251e 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -69,13 +69,13 @@
 } LZ4F_frameType_t;
 


 
typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
-  unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
-  unsigned               dictID;                /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
-  unsigned               reserved[1];           /* must be zero for forward compatibility */
+  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  LZ4F_blockMode_t       blockMode;            /* blockLinked, blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
+  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;               /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               reserved[1];          /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 makes it possible to supply detailed frame parameters to the stream interface.
  It's not required to set all fields, as long as the structure was initially memset() to zero.
@@ -85,7 +85,7 @@
 
typedef struct {
   LZ4F_frameInfo_t frameInfo;
   int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
-  unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
+  unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
 makes it possible to supply detailed compression parameters to the stream interface.
@@ -101,12 +101,12 @@
  
 


 
-
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-
 Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
- An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
- This value is supplied by LZ4F_compressFrameBound().
- If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
- The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+                                const void* srcBuffer, size_t srcSize,
+                                const LZ4F_preferences_t* preferencesPtr);
+
  Compress an entire srcBuffer into a valid LZ4 frame.
+  dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+  The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
  @return : number of bytes written into dstBuffer.
            or an error code if it fails (can be tested using LZ4F_isError())
  
@@ -134,9 +134,11 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 
Compression

 
-
size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
-
 will write the frame header into dstBuffer.
- dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+
size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
+                                      void* dstBuffer, size_t dstCapacity,
+                                      const LZ4F_preferences_t* prefsPtr);
+
  will write the frame header into dstBuffer.
+  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
  `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
  @return : number of bytes written into dstBuffer for the header
            or an error code (which can be tested using LZ4F_isError())
diff --git a/lib/lz4.c b/lib/lz4.c
index 87ec6ab..69ee3eb 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -938,6 +938,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 
 int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
 {
+    if (!LZ4_stream) return 0;   /* support free on NULL */
     FREEMEM(LZ4_stream);
     return (0);
 }
@@ -1277,11 +1278,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 
 /*===== streaming decompression functions =====*/
 
-/*
- * If you prefer dynamic allocation methods,
- * LZ4_createStreamDecode()
- * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
- */
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 {
     LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
@@ -1290,6 +1286,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 
 int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
 {
+    if (!LZ4_stream) return 0;   /* support free on NULL */
     FREEMEM(LZ4_stream);
     return 0;
 }
diff --git a/lib/lz4.h b/lib/lz4.h
index 5b6bc92..c5b6103 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -380,7 +380,7 @@ typedef struct {
  * init this structure before first use.
  * note : only use in association with static linking !
  *        this definition is not API/ABI safe,
- *        and may change in a future version !
+ *        it may change in a future version !
  */
 #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
 #define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index c2d6d5c..f9af991 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -170,6 +170,7 @@ typedef struct LZ4F_cctx_s
     LZ4F_preferences_t prefs;
     U32    version;
     U32    cStage;
+    int    dictionary;
     size_t maxBlockSize;
     size_t maxBufferSize;
     BYTE*  tmpBuff;
@@ -178,7 +179,7 @@ typedef struct LZ4F_cctx_s
     U64    totalInSize;
     XXH32_state_t xxh;
     void*  lz4CtxPtr;
-    U32    lz4CtxLevel;     /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
+    U32    lz4CtxLevel;   /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
 } LZ4F_cctx_t;
 
 
@@ -256,7 +257,7 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
     return requestedBSID;
 }
 
-/* LZ4F_compressBound() :
+/* LZ4F_compressBound_internal() :
  * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
  * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
  * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
@@ -301,16 +302,19 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
 }
 
 
-/*! LZ4F_compressFrame() :
- *  Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
- *  The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
- *  If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
- *  Get the minimum value of dstCapacity by using LZ4F_compressFrameBound().
- *  The LZ4F_preferences_t structure is optional : if NULL is provided as argument, preferences will be set to default.
- *  The result of the function is the number of bytes written into dstBuffer.
- *  The function outputs an error code if it fails (can be tested using LZ4F_isError())
+/*! LZ4F_compressFrame_usingCDict() :
+ *  Compress srcBuffer using a dictionary, in a single step.
+ *  cdict can be NULL, in which case, no dictionary is used.
+ *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ *  however, it's the only way to provide a dictID, so it's not recommended.
+ * @return : number of bytes written into dstBuffer.
+ *           or an error code if it fails (can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
+size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
+                               const void* srcBuffer, size_t srcSize,
+                               const LZ4F_CDict* cdict,
+                               const LZ4F_preferences_t* preferencesPtr)
 {
     LZ4F_cctx_t cctxI;
     LZ4_stream_t lz4ctx;
@@ -321,10 +325,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
     BYTE* const dstEnd = dstStart + dstCapacity;
 
     memset(&cctxI, 0, sizeof(cctxI));
-    memset(&options, 0, sizeof(options));
-
     cctxI.version = LZ4F_VERSION;
-    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works because autoflush==1 & stableSrc==1 */
+    cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
+    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+        cctxI.lz4CtxPtr = &lz4ctx;
+        cctxI.lz4CtxLevel = 1;
+    }  /* fast compression context pre-created on stack */
 
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
@@ -333,22 +339,18 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
     if (prefs.frameInfo.contentSize != 0)
         prefs.frameInfo.contentSize = (U64)srcSize;   /* auto-correct content size if selected (!=0) */
 
-    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-        cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 1;
-    }  /* otherwise : will be created within LZ4F_compressBegin */
-
     prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
     prefs.autoFlush = 1;
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
-        prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* no need for linked blocks */
+        prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
+    memset(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
-    if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))   /* condition to guarantee success */
+    if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs))  /* condition to guarantee success */
         return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
 
-    { size_t const headerSize = LZ4F_compressBegin(&cctxI, dstBuffer, dstCapacity, &prefs);  /* write header */
+    { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs);  /* write header */
       if (LZ4F_isError(headerSize)) return headerSize;
       dstPtr += headerSize;   /* header size */ }
 
@@ -360,24 +362,91 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
       if (LZ4F_isError(tailSize)) return tailSize;
       dstPtr += tailSize; }
 
-    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)   /* Ctx allocation only for lz4hc */
+    if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN)  /* Ctx allocation only for lz4hc */
         FREEMEM(cctxI.lz4CtxPtr);
 
     return (dstPtr - dstStart);
 }
 
 
+/*! LZ4F_compressFrame() :
+ *  Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
+ *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ * @return : number of bytes written into dstBuffer.
+ *           or an error code if it fails (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+                    const void* srcBuffer, size_t srcSize,
+                    const LZ4F_preferences_t* preferencesPtr)
+{
+    return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
+                                         srcBuffer, srcSize,
+                                         NULL, preferencesPtr);
+}
+
+
+/*-***************************************************
+*   Dictionary compression
+*****************************************************/
+
+struct LZ4F_CDict_s {
+    void* dictContent;
+    LZ4_stream_t* fastCtx;
+    LZ4_streamHC_t* HCCtx;
+}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
+
+/*! LZ4_createCDict() :
+ *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict
+ * @return : digested dictionary for compression, or NULL if failed */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
+{
+    const char* dictStart = (const char*)dictBuffer;
+    LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
+    if (!cdict) return NULL;
+    if (dictSize > 64 KB) {
+        dictStart += dictSize - 64 KB;
+        dictSize = 64 KB;
+    }
+    cdict->dictContent = ALLOCATOR(dictSize);
+    cdict->fastCtx = LZ4_createStream();
+    cdict->HCCtx = LZ4_createStreamHC();
+    if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
+        LZ4F_freeCDict(cdict);
+        return NULL;
+    }
+    memcpy(cdict->dictContent, dictStart, dictSize);
+    LZ4_resetStream(cdict->fastCtx);
+    LZ4_loadDict (cdict->fastCtx, cdict->dictContent, (int)dictSize);
+    LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
+    LZ4_loadDictHC(cdict->HCCtx, cdict->dictContent, (int)dictSize);
+    return cdict;
+}
+
+void LZ4F_freeCDict(LZ4F_CDict* cdict)
+{
+    if (cdict==NULL) return;  /* support free on NULL */
+    FREEMEM(cdict->dictContent);
+    LZ4_freeStream(cdict->fastCtx);
+    LZ4_freeStreamHC(cdict->HCCtx);
+    FREEMEM(cdict);
+}
+
+
 /*-*********************************
 *  Advanced compression functions
 ***********************************/
 
 /*! LZ4F_createCompressionContext() :
- * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
- * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
- * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries.
- * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
- * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
- * Object can release its memory using LZ4F_freeCompressionContext();
+ *  The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ *  This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ *  The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
+ *  The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
+ *  If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
+ *  Object can release its memory using LZ4F_freeCompressionContext();
  */
 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
 {
@@ -385,7 +454,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c
     if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
 
     cctxPtr->version = version;
-    cctxPtr->cStage = 0;   /* Next stage : write header */
+    cctxPtr->cStage = 0;   /* Next stage : init stream */
 
     *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
 
@@ -397,8 +466,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 {
     LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
 
-    if (cctxPtr != NULL) {  /* null pointers can be safely provided to this function, like free() */
-       FREEMEM(cctxPtr->lz4CtxPtr);
+    if (cctxPtr != NULL) {  /* support free on NULL */
+       FREEMEM(cctxPtr->lz4CtxPtr);  /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
        FREEMEM(cctxPtr->tmpBuff);
        FREEMEM(LZ4F_compressionContext);
     }
@@ -407,22 +476,23 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 }
 
 
-/*! LZ4F_compressBegin() :
- *  will write the frame header into dstBuffer.
- *  dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
- *  @return : number of bytes written into dstBuffer for the header
- *            or an error code (can be tested using LZ4F_isError())
+/*! LZ4F_compressBegin_usingCDict() :
+ *  init streaming compression and writes frame header into dstBuffer.
+ *  dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * @return : number of bytes written into dstBuffer for the header
+ *           or an error code (can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr)
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
+                          void* dstBuffer, size_t dstCapacity,
+                          const LZ4F_CDict* cdict,
+                          const LZ4F_preferences_t* preferencesPtr)
 {
     LZ4F_preferences_t prefNull;
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     BYTE* headerStart;
-    size_t requiredBuffSize;
 
     if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
-    if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC);
     memset(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
@@ -437,31 +507,41 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
             cctxPtr->lz4CtxLevel = tableID;
-        }
-    }
+    }   }
 
     /* Buffer Management */
-    if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
+    if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
+        cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
     cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
 
-    requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
-    if (preferencesPtr->autoFlush)
-        requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB;   /* just needs dict */
+    {   size_t const requiredBuffSize = preferencesPtr->autoFlush ?
+                (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB :  /* only needs windows size */
+                cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
 
-    if (cctxPtr->maxBufferSize < requiredBuffSize) {
-        cctxPtr->maxBufferSize = 0;
-        FREEMEM(cctxPtr->tmpBuff);
-        cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
-        if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-        cctxPtr->maxBufferSize = requiredBuffSize;
-    }
+        if (cctxPtr->maxBufferSize < requiredBuffSize) {
+            cctxPtr->maxBufferSize = 0;
+            FREEMEM(cctxPtr->tmpBuff);
+            cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
+            if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+            cctxPtr->maxBufferSize = requiredBuffSize;
+    }   }
     cctxPtr->tmpIn = cctxPtr->tmpBuff;
     cctxPtr->tmpInSize = 0;
     XXH32_reset(&(cctxPtr->xxh), 0);
-    if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-        LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-    else
-        LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+
+    /* context init */
+    cctxPtr->dictionary = (cdict!=NULL);
+    if (cdict) {
+        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+            memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        } else
+            memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+    } else {
+        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+            LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+        else
+            LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
+    }
 
     /* Magic Number */
     LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
@@ -487,16 +567,31 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
         LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
         dstPtr += 4;
     }
-    /* CRC Byte */
+    /* Header CRC Byte */
     *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
     dstPtr++;
 
     cctxPtr->cStage = 1;   /* header written, now request input data block */
-
     return (dstPtr - dstStart);
 }
 
 
+/*! LZ4F_compressBegin() :
+ *  init streaming compression and writes frame header into dstBuffer.
+ *  dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ *  preferencesPtr can be NULL, in which case default parameters are selected.
+ * @return : number of bytes written into dstBuffer for the header
+ *           or an error code (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
+                          void* dstBuffer, size_t dstCapacity,
+                          const LZ4F_preferences_t* preferencesPtr)
+{
+    return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
+                                         NULL, preferencesPtr);
+}
+
+
 /* LZ4F_compressBound() :
  *      @ return size of Dst buffer given a srcSize to handle worst case situations.
  *      The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations.
@@ -537,19 +632,19 @@ static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char*
     return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level)
+static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
 {
     (void) level;
-    return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize);
+    return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
 }
 
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int dictionary, int level)
 {
     if (level < LZ4HC_CLEVEL_MIN) {
-        if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
+        if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4F_localLZ4_compress_limitedOutput_withState;
         return LZ4F_localLZ4_compress_limitedOutput_continue;
     }
-    if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC;
+    if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4_compress_HC_extStateHC;
     return LZ4F_localLZ4_compressHC_limitedOutput_continue;
 }
 
@@ -580,7 +675,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
-    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
+    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
@@ -679,7 +774,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
     (void)compressOptionsPtr;   /* not yet useful */
 
     /* select compression function */
-    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
+    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
 
     /* compress tmp buffer */
     dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 0efe220..42e2f53 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -157,13 +157,13 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
  * It's not required to set all fields, as long as the structure was initially memset() to zero.
  * All reserved fields must be set to zero. */
 typedef struct {
-  LZ4F_blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
-  unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
-  unsigned               dictID;                /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
-  unsigned               reserved[1];           /* must be zero for forward compatibility */
+  LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  LZ4F_blockMode_t       blockMode;            /* blockLinked, blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
+  unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
+  unsigned               dictID;               /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               reserved[1];          /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
@@ -173,7 +173,7 @@ typedef struct {
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
   int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
-  unsigned autoFlush;              /* 1 == always flush (reduce usage of tmp buffer) */
+  unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
@@ -187,17 +187,16 @@ typedef struct {
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
-/*!LZ4F_compressFrame() :
- * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
- * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation.
- * This value is supplied by LZ4F_compressFrameBound().
- * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode).
- * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+/*! LZ4F_compressFrame() :
+ *  Compress an entire srcBuffer into a valid LZ4 frame.
+ *  dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
  * @return : number of bytes written into dstBuffer.
  *           or an error code if it fails (can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-
+LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
+                                const void* srcBuffer, size_t srcSize,
+                                const LZ4F_preferences_t* preferencesPtr);
 
 
 /*-***********************************
@@ -231,13 +230,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 #define LZ4F_HEADER_SIZE_MAX 19
 /*! LZ4F_compressBegin() :
- * will write the frame header into dstBuffer.
- * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
+ *  will write the frame header into dstBuffer.
+ *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
  * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
  * @return : number of bytes written into dstBuffer for the header
  *           or an error code (which can be tested using LZ4F_isError())
  */
-LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
+                                      void* dstBuffer, size_t dstCapacity,
+                                      const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
  * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index d3bae82..5f22aed 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -82,6 +82,50 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
 LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
 
 
+
+/**********************************
+ *  Bulk processing dictionary API
+ *********************************/
+typedef struct LZ4F_CDict_s LZ4F_CDict;
+
+/*! LZ4_createCDict() :
+ *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ *  `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+
+/*! LZ4_compressFrame_usingCDict() :
+ *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
+ *  If cdict==NULL, compress without a dictionary.
+ *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
+ *  If this condition is not respected, function will fail (@return an errorCode).
+ *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
+ *  but it's not recommended, as it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer.
+ *           or an error code if it fails (can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
+                               const void* src, size_t srcSize,
+                               const LZ4F_CDict* cdict,
+                               const LZ4F_preferences_t* preferencesPtr);
+
+
+/*! LZ4F_compressBegin_usingCDict() :
+ *  Inits streaming dictionary compression, and writes the frame header into dstBuffer.
+ *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
+ * `prefsPtr` is optional : you may provide NULL as argument,
+ *  however, it's the only way to provide dictID in the frame header.
+ * @return : number of bytes written into dstBuffer for the header,
+ *           or an error code (which can be tested using LZ4F_isError())
+ */
+size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
+                                     void* dstBuffer, size_t dstCapacity,
+                                     const LZ4F_CDict* cdict,
+                                     const LZ4F_preferences_t* prefsPtr);
+
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index ca9c2e6..b17760d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -610,7 +610,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i
 **************************************/
 /* allocation */
 LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
-int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
+int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) {
+    if (!LZ4_streamHCPtr) return 0;  /* support free on NULL */
+    free(LZ4_streamHCPtr);
+    return 0;
+}
 
 
 /* initialization */
@@ -767,7 +771,11 @@ void* LZ4_createHC (char* inputBuffer)
     return hc4;
 }
 
-int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; }
+int LZ4_freeHC (void* LZ4HC_Data) {
+    if (!LZ4HC_Data) return 0;  /* support free on NULL */
+    FREEMEM(LZ4HC_Data);
+    return 0;
+}
 
 int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel)
 {
diff --git a/tests/frametest.c b/tests/frametest.c
index 02d7e5b..1ee0d96 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -482,10 +482,36 @@ int basicTests(U32 seed, double compressibility)
         if (prefs.frameInfo.dictID != dictID) goto _output_error;
         DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
 
-        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
         CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
+        CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
+    }
+
+
+    /* Dictionary compression test */
+    {   size_t const dictSize = 63 KB;
+        size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
+        size_t cSizeNoDict, cSizeWithDict;
+        LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
+        if (cdict == NULL) goto _output_error;
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
+        CHECK_V(cSizeNoDict,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                                              CNBuffer, dictSize,
+                                              NULL, NULL) );
+        DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
+
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
+        CHECK_V(cSizeWithDict,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                                              CNBuffer, dictSize,
+                                              cdict, NULL) );
+        DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeWithDict);
+        if (cSizeWithDict >= cSizeNoDict) goto _output_error;  /* must be more efficient */
+
+        LZ4F_freeCDict(cdict);
     }
 
+
     DISPLAYLEVEL(3, "Skippable frame test : \n");
     {   size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
         unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
-- 
cgit v0.12


From bf8daa2fd52fe4a87b59f74e3f3c249f57218391 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 9 Aug 2017 18:00:48 -0700
Subject: fixed uninitialization error in lz4frame

---
 lib/lz4frame.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index f9af991..07cbf56 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -327,10 +327,6 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     memset(&cctxI, 0, sizeof(cctxI));
     cctxI.version = LZ4F_VERSION;
     cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
-    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-        cctxI.lz4CtxPtr = &lz4ctx;
-        cctxI.lz4CtxLevel = 1;
-    }  /* fast compression context pre-created on stack */
 
     if (preferencesPtr!=NULL)
         prefs = *preferencesPtr;
@@ -344,6 +340,11 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = LZ4F_blockIndependent;   /* only one block => no need for inter-block link */
 
+    if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+        cctxI.lz4CtxPtr = &lz4ctx;
+        cctxI.lz4CtxLevel = 1;
+    }  /* fast compression context pre-created on stack */
+
     memset(&options, 0, sizeof(options));
     options.stableSrc = 1;
 
-- 
cgit v0.12


From ca2fb166ab650835c71099842b646d388cebb3ac Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 9 Aug 2017 18:22:26 -0700
Subject: fixed C++ conversion warnings

---
 lib/lz4frame.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 07cbf56..180658d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -421,9 +421,9 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
     }
     memcpy(cdict->dictContent, dictStart, dictSize);
     LZ4_resetStream(cdict->fastCtx);
-    LZ4_loadDict (cdict->fastCtx, cdict->dictContent, (int)dictSize);
+    LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
     LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
-    LZ4_loadDictHC(cdict->HCCtx, cdict->dictContent, (int)dictSize);
+    LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
     return cdict;
 }
 
-- 
cgit v0.12


From d8aafe2c52b4b16da9e8d289ad9bdb04b0bc4a05 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 10 Aug 2017 00:48:19 -0700
Subject: dictionary compression correctly uses compression level

Not obvious : copying the state was copying cdict's compression level
---
 doc/lz4_manual.html      | 36 ++++++++++++++++++++----------------
 doc/lz4frame_manual.html |  2 +-
 lib/lz4.h                | 36 ++++++++++++++++++------------------
 lib/lz4frame.c           | 23 +++++++++++++----------
 lib/lz4frame.h           |  4 +++-
 lib/lz4hc.c              | 12 ++++++++++++
 lib/lz4hc.h              | 23 ++++++++++++++++-------
 tests/frametest.c        | 24 ++++++++++++++++++++++++
 8 files changed, 107 insertions(+), 53 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 60b41a8..ecc985d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -175,8 +175,8 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
   Important : Previous data blocks are assumed to remain present and unmodified !
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
-  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
-  After an error, the stream status is invalid, and it can only be reset or freed.
+  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+  After an error, the stream status is invalid, it can only be reset or freed.
  
 


 
@@ -205,20 +205,20 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
 
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-
    These decoding functions allow decompression of multiple blocks in "streaming" mode.
-    Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
-    In the case of a ring buffers, decoding buffer must be either :
-    - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
-      In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
-    - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
-      maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
-      In which case, encoding and decoding buffers do not need to be synchronized,
-      and encoding ring buffer can have any size, including small ones ( < 64 KB).
-    - _At least_ 64 KB + 8 bytes + maxBlockSize.
-      In which case, encoding and decoding buffers do not need to be synchronized,
-      and encoding ring buffer can have any size, including larger than decoding buffer.
-    Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-    and indicate where it is saved using LZ4_setStreamDecode()
+
  These decoding functions allow decompression of multiple blocks in "streaming" mode.
+  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+  In the case of a ring buffers, decoding buffer must be either :
+  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+    maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+    In which case, encoding and decoding buffers do not need to be synchronized,
+    and encoding ring buffer can have any size, including small ones ( < 64 KB).
+  - _At least_ 64 KB + 8 bytes + maxBlockSize.
+    In which case, encoding and decoding buffers do not need to be synchronized,
+    and encoding ring buffer can have any size, including larger than decoding buffer.
+  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+  and indicate where it is saved using LZ4_setStreamDecode()
 


 
 
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
@@ -321,5 +321,9 @@ union LZ4_streamDecode_u {
    Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS 
 


 
+
Experimental API section
#ifdef LZ4_HC_STATIC_LINKING_ONLY
+#ifndef LZ4_SLO_299387928743
+#define LZ4_SLO_299387928743
+


 
 
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index dc8251e..a1a6e96 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -74,7 +74,7 @@
   LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
   LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
   unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
   unsigned               reserved[1];          /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 makes it possible to supply detailed frame parameters to the stream interface.
diff --git a/lib/lz4.h b/lib/lz4.h
index c5b6103..86ca0d5 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -259,8 +259,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  *  Important : Previous data blocks are assumed to remain present and unmodified !
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- *  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
- *  After an error, the stream status is invalid, and it can only be reset or freed.
+ *  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+ *  After an error, the stream status is invalid, it can only be reset or freed.
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
@@ -291,22 +291,21 @@ LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 
-/*!
-LZ4_decompress_*_continue() :
-    These decoding functions allow decompression of multiple blocks in "streaming" mode.
-    Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
-    In the case of a ring buffers, decoding buffer must be either :
-    - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
-      In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
-    - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
-      maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
-      In which case, encoding and decoding buffers do not need to be synchronized,
-      and encoding ring buffer can have any size, including small ones ( < 64 KB).
-    - _At least_ 64 KB + 8 bytes + maxBlockSize.
-      In which case, encoding and decoding buffers do not need to be synchronized,
-      and encoding ring buffer can have any size, including larger than decoding buffer.
-    Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-    and indicate where it is saved using LZ4_setStreamDecode()
+/*! LZ4_decompress_*_continue() :
+ *  These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ *  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+ *  In the case of a ring buffers, decoding buffer must be either :
+ *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ *    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ *  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ *    maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ *    In which case, encoding and decoding buffers do not need to be synchronized,
+ *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ *  - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ *    In which case, encoding and decoding buffers do not need to be synchronized,
+ *    and encoding ring buffer can have any size, including larger than decoding buffer.
+ *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ *  and indicate where it is saved using LZ4_setStreamDecode()
 */
 LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
 LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
@@ -458,6 +457,7 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4
 
 #endif /* LZ4_H_2983827168210 */
 
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 180658d..9aeb456 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -61,6 +61,7 @@ You can contact the author at :
 **************************************/
 #include "lz4frame_static.h"
 #include "lz4.h"
+#define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"
@@ -217,6 +218,8 @@ static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
 
 unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
 
+int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
+
 
 /*-************************************
 *  Private functions
@@ -308,7 +311,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
  *  dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
  *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
  *  however, it's the only way to provide a dictID, so it's not recommended.
- * @return : number of bytes written into dstBuffer.
+ * @return : number of bytes written into dstBuffer,
  *           or an error code if it fails (can be tested using LZ4F_isError())
  */
 size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
@@ -535,8 +538,10 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     if (cdict) {
         if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
             memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
-        } else
+        } else {
             memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+            LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+        }
     } else {
         if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
             LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
@@ -659,14 +664,12 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 
 /*! LZ4F_compressUpdate() :
-* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
-* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case.
-* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode)
-* You can get the minimum value of dstCapacity by using LZ4F_compressBound()
-* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
-* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
-* The function outputs an error code if it fails (can be tested using LZ4F_isError())
-*/
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ *  LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ */
 size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
 {
     LZ4F_compressOptions_t cOptionsNull;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 42e2f53..307d7b2 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -162,7 +162,7 @@ typedef struct {
   LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
   LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
   unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
-  unsigned               dictID;               /* Dictionary ID, sent by the compressor, to help the decoder select the right dictionary; 0 == no dictionary used */
+  unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
   unsigned               reserved[1];          /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
@@ -177,6 +177,8 @@ typedef struct {
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
+LZ4FLIB_API int LZ4F_compressionLevel_max();
+
 
 /*-*********************************
 *  Simple compression function
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b17760d..22eb071 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -69,6 +69,8 @@
 
 
 /*===   Macros   ===*/
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+#define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define HASH_FUNCTION(i)         (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
 #define DELTANEXTMAXD(p)         chainTable[(p) & LZ4HC_MAXD_MASK]    /* flexible, LZ4HC_MAXD dependent */
 #define DELTANEXTU16(table, pos) table[(U16)(pos)]   /* faster */
@@ -627,6 +629,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
 }
 
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+    int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel;
+    int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN;
+    int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX;
+    compressionLevel = MIN(compressionLevel, minCLevel);
+    compressionLevel = MAX(compressionLevel, maxCLevel);
+    LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+}
+
 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
 {
     LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index b63c825..66d5636 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -161,7 +161,7 @@ typedef struct
 typedef struct
 {
     unsigned int   hashTable[LZ4HC_HASHTABLESIZE];
-    unsigned short   chainTable[LZ4HC_MAXD];
+    unsigned short chainTable[LZ4HC_MAXD];
     const unsigned char* end;        /* next block here to continue on current prefix */
     const unsigned char* base;       /* All index relative to this position */
     const unsigned char* dictBase;   /* alternate base for extDict */
@@ -245,25 +245,34 @@ LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStr
  *           or 0 if compression fails.
  * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
  */
-LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data,
+int LZ4_compress_HC_destSize(void* LZ4HC_Data,
                             const char* src, char* dst,
                             int* srcSizePtr, int targetDstSize,
                             int compressionLevel);
 
-/*! LZ4_compress_HC_continue_destSize() :
+/*! LZ4_compress_HC_continue_destSize() : v1.8.0 (experimental)
  *  Similar as LZ4_compress_HC_continue(),
  *  but will read a variable nb of bytes from `src`
  *  to fit into `targetDstSize` budget.
  *  Result is provided in 2 parts :
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
- * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`
- * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
- *             beyond that level, compression performance will be much reduced due to internal incompatibilities
+ * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src`.
+ *  Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN
+ *              beyond that level, compression performance will be much reduced due to internal incompatibilities
  */
-LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
+int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
                             const char* src, char* dst,
                             int* srcSizePtr, int targetDstSize);
 
+/*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
+ *  It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(),
+ *  but that requires to stay in the same mode (aka 1-10 or 11-12).
+ *  This function ensures this condition.
+ */
+void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
+
+
+
 #endif   /* LZ4_HC_SLO_098092834 */
 #endif   /* LZ4_HC_STATIC_LINKING_ONLY */
diff --git a/tests/frametest.c b/tests/frametest.c
index 1ee0d96..4723aac 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -508,6 +508,30 @@ int basicTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeWithDict);
         if (cSizeWithDict >= cSizeNoDict) goto _output_error;  /* must be more efficient */
 
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
+        {   size_t cSizeLevelMax;
+            LZ4F_preferences_t cParams;
+            memset(&cParams, 0, sizeof(cParams));
+            cParams.compressionLevel = -3;
+            CHECK_V(cSizeLevelMax,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                                              CNBuffer, dictSize,
+                                              cdict, &cParams) );
+            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
+        }
+
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
+        {   size_t cSizeLevelMax;
+            LZ4F_preferences_t cParams;
+            memset(&cParams, 0, sizeof(cParams));
+            cParams.compressionLevel = LZ4F_compressionLevel_max();
+            CHECK_V(cSizeLevelMax,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
+                                              CNBuffer, dictSize,
+                                              cdict, &cParams) );
+            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
+        }
+
         LZ4F_freeCDict(cdict);
     }
 
-- 
cgit v0.12


From 8d597d62d5455f5d315782048b9d32f129a60bac Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 10 Aug 2017 10:28:52 -0700
Subject: fixed gcc prototype warning

---
 doc/lz4_manual.html | 4 ----
 lib/lz4frame.h      | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index ecc985d..9ab1984 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -321,9 +321,5 @@ union LZ4_streamDecode_u {
    Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS 
 


 
-
Experimental API section
#ifdef LZ4_HC_STATIC_LINKING_ONLY
-#ifndef LZ4_SLO_299387928743
-#define LZ4_SLO_299387928743
-


 
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 307d7b2..ccacb89 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -177,7 +177,7 @@ typedef struct {
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
-LZ4FLIB_API int LZ4F_compressionLevel_max();
+LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 
 
 /*-*********************************
-- 
cgit v0.12


From 4531637ecdc4b12154c66cc69cfaade185039e4c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 10 Aug 2017 12:12:53 -0700
Subject: support dictionary compression with independent blocks

---
 doc/lz4frame_manual.html | 55 ++++++++++++++++----------------
 lib/lz4frame.c           | 82 ++++++++++++++++++++++++++++++------------------
 lib/lz4frame.h           | 63 +++++++++++++++++++------------------
 tests/frametest.c        | 30 ++++++++++++++++++
 4 files changed, 141 insertions(+), 89 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index a1a6e96..9593181 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -96,8 +96,8 @@
 
Simple compression function

 
 
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-
 Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+
  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
  
 


 
@@ -210,10 +210,10 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
-
 This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- Its usage is optional.
- Extracted information can typically be useful for allocation purposes.
- This function works in 2 situations :
+
  This function extracts frame parameters (max blockSize, dictID, etc.).
+  Its usage is optional.
+  Extracted information is typically useful for allocation and dictionary.
+  This function works in 2 situations :
    - At the beginning of a new frame, in which case
      it will decode information from `srcBuffer`, starting the decoding process.
      Input size must be large enough to successfully decode the entire frame header.
@@ -221,13 +221,14 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
      It's allowed to provide more input data than this minimum.
    - After decoding has been started.
      In which case, no input is read, frame parameters are extracted from dctx.
-   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- Decompression must resume from (srcBuffer + *srcSizePtr).
+   - If decoding has barely started, but not yet extracted information from header,
+     LZ4F_getFrameInfo() will fail.
+  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+  Decompression must resume from (srcBuffer + *srcSizePtr).
  @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
-           or an error code which can be tested using LZ4F_isError()
- note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+           or an error code which can be tested using LZ4F_isError().
+  note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  
 


 
@@ -235,28 +236,28 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                    void* dstBuffer, size_t* dstSizePtr,
                                    const void* srcBuffer, size_t* srcSizePtr,
                                    const LZ4F_decompressOptions_t* dOptPtr);
-
 Call this function repetitively to regenerate compressed data from `srcBuffer`.
- The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+
  Call this function repetitively to regenerate compressed data from `srcBuffer`.
+  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
 
- The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+  The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
 
- The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- Number of bytes consumed can be < number of bytes provided.
- It typically happens when dstBuffer is not large enough to contain all decoded data.
- Unconsumed source data must be presented again in subsequent invocations.
+  The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+  Number of bytes consumed can be < number of bytes provided.
+  It typically happens when dstBuffer is not large enough to contain all decoded data.
+  Unconsumed source data must be presented again in subsequent invocations.
 
  `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
  `dstBuffer` itself can be changed at will between each consecutive function invocation.
 
- @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- This is just a hint though, it's always possible to provide any srcSize.
- When a frame is fully decoded, @return will be 0 (no more data expected).
- If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+  Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+  This is just a hint though, it's always possible to provide any srcSize.
+  When a frame is fully decoded, @return will be 0 (no more data expected).
+  If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
 
- After a frame is fully decoded, dctx can be used again to decompress another frame.
- After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+  After a frame is fully decoded, dctx can be used again to decompress another frame.
+  After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  
 


 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 9aeb456..fbccc9d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -171,7 +171,7 @@ typedef struct LZ4F_cctx_s
     LZ4F_preferences_t prefs;
     U32    version;
     U32    cStage;
-    int    dictionary;
+    const LZ4F_CDict* cdict;
     size_t maxBlockSize;
     size_t maxBufferSize;
     BYTE*  tmpBuff;
@@ -534,19 +534,22 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     XXH32_reset(&(cctxPtr->xxh), 0);
 
     /* context init */
-    cctxPtr->dictionary = (cdict!=NULL);
-    if (cdict) {
-        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-            memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+    cctxPtr->cdict = cdict;
+    if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
+        /* frame init only for blockLinked : blockIndependent will be init at each block */
+        if (cdict) {
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
+                memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
+            } else {
+                memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
+                LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            }
         } else {
-            memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
-            LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+                LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
+            else
+                LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
         }
-    } else {
-        if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-            LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
-        else
-            LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
     }
 
     /* Magic Number */
@@ -609,13 +612,15 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
 }
 
 
-typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level);
+typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
 
-static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level)
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict)
 {
     /* compress a single block */
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level);
+    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
+                                      (int)(srcSize), (int)(srcSize-1),
+                                      level, cdict);
     LZ4F_writeLE32(cSizePtr, cSize);
     if (cSize == 0) {  /* compression failed */
         cSize = (U32)srcSize;
@@ -626,32 +631,47 @@ static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, com
 }
 
 
-static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    if (cdict) {
+        memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
+        return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
+    }
     return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
-static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
     int const acceleration = (level < -1) ? -level : 1;
+    (void)cdict; /* init once at beginning of frame */
     return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
 }
 
-static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level)
+static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
+{
+    if (cdict) {
+        memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
+        LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
+        return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
+    }
+    return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
+}
+
+static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
 {
-    (void) level;
+    (void)level; (void)cdict; /* init once at beginning of frame */
     return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
 }
 
-static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int dictionary, int level)
+static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
 {
     if (level < LZ4HC_CLEVEL_MIN) {
-        if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4F_localLZ4_compress_limitedOutput_withState;
-        return LZ4F_localLZ4_compress_limitedOutput_continue;
+        if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
+        return LZ4F_compressBlock_continue;
     }
-    if (blockMode == LZ4F_blockIndependent && !dictionary) return LZ4_compress_HC_extStateHC;
-    return LZ4F_localLZ4_compressHC_limitedOutput_continue;
+    if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
+    return LZ4F_compressBlockHC_continue;
 }
 
 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
@@ -679,7 +699,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
-    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+    compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
@@ -702,7 +722,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
             memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
             srcPtr += sizeToCopy;
 
-            dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+            dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
 
             if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
             cctxPtr->tmpInSize = 0;
@@ -710,16 +730,16 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     }
 
     while ((size_t)(srcEnd - srcPtr) >= blockSize) {
-        /* compress full block */
+        /* compress full blocks */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
         srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
         /* compress remaining input < blockSize */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
         srcPtr  = srcEnd;
     }
 
@@ -778,10 +798,10 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
     (void)compressOptionsPtr;   /* not yet useful */
 
     /* select compression function */
-    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->dictionary, cctxPtr->prefs.compressionLevel);
+    compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
     /* compress tmp buffer */
-    dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
+    dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
     if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
     cctxPtr->tmpInSize = 0;
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index ccacb89..dd76194 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -100,8 +100,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /**< return
 /*-************************************
  *  Frame compression types
  **************************************/
-/* #define LZ4F_DISABLE_OBSOLETE_ENUMS */  /* uncomment to disable obsolete enums */
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+/* #define LZ4F_ENABLE_OBSOLETE_ENUMS   // uncomment to enable obsolete enums */
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 #  define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
 #else
 #  define LZ4F_OBSOLETE_ENUM(x)
@@ -145,7 +145,7 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 
-#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 typedef LZ4F_blockSizeID_t blockSizeID_t;
 typedef LZ4F_blockMode_t blockMode_t;
 typedef LZ4F_frameType_t frameType_t;
@@ -183,9 +183,9 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 /*-*********************************
 *  Simple compression function
 ***********************************/
-/*!LZ4F_compressFrameBound() :
- * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
- * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+/*! LZ4F_compressFrameBound() :
+ *  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ *  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
@@ -316,10 +316,10 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 *************************************/
 
 /*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- * Its usage is optional.
- * Extracted information can typically be useful for allocation purposes.
- * This function works in 2 situations :
+ *  This function extracts frame parameters (max blockSize, dictID, etc.).
+ *  Its usage is optional.
+ *  Extracted information is typically useful for allocation and dictionary.
+ *  This function works in 2 situations :
  *   - At the beginning of a new frame, in which case
  *     it will decode information from `srcBuffer`, starting the decoding process.
  *     Input size must be large enough to successfully decode the entire frame header.
@@ -327,41 +327,42 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
  *     It's allowed to provide more input data than this minimum.
  *   - After decoding has been started.
  *     In which case, no input is read, frame parameters are extracted from dctx.
- *   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ *   - If decoding has barely started, but not yet extracted information from header,
+ *     LZ4F_getFrameInfo() will fail.
+ *  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ *  Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
- *           or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ *           or an error code which can be tested using LZ4F_isError().
+ *  note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
+ *  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  */
 LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
 /*! LZ4F_decompress() :
- * Call this function repetitively to regenerate compressed data from `srcBuffer`.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *  Call this function repetitively to regenerate compressed data from `srcBuffer`.
+ *  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
  *
- * The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ *  The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
  *
- * The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- * Number of bytes consumed can be < number of bytes provided.
- * It typically happens when dstBuffer is not large enough to contain all decoded data.
- * Unconsumed source data must be presented again in subsequent invocations.
+ *  The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ *  Number of bytes consumed can be < number of bytes provided.
+ *  It typically happens when dstBuffer is not large enough to contain all decoded data.
+ *  Unconsumed source data must be presented again in subsequent invocations.
  *
  * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
  * `dstBuffer` itself can be changed at will between each consecutive function invocation.
  *
- * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
- * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
- * This is just a hint though, it's always possible to provide any srcSize.
- * When a frame is fully decoded, @return will be 0 (no more data expected).
- * If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
+ *  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ *  Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
+ *  This is just a hint though, it's always possible to provide any srcSize.
+ *  When a frame is fully decoded, @return will be 0 (no more data expected).
+ *  If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
  *
- * After a frame is fully decoded, dctx can be used again to decompress another frame.
- * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
+ *  After a frame is fully decoded, dctx can be used again to decompress another frame.
+ *  After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  */
 LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    void* dstBuffer, size_t* dstSizePtr,
diff --git a/tests/frametest.c b/tests/frametest.c
index 4723aac..a30089f 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -532,6 +532,36 @@ int basicTests(U32 seed, double compressibility)
             DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
         }
 
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
+        {   size_t cSizeContiguous;
+            size_t const inSize = dictSize * 3;
+            size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
+            LZ4F_preferences_t cParams;
+            memset(&cParams, 0, sizeof(cParams));
+            cParams.frameInfo.blockMode = LZ4F_blockLinked;
+            cParams.frameInfo.blockSizeID = LZ4F_max64KB;
+            CHECK_V(cSizeContiguous,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                                              CNBuffer, inSize,
+                                              cdict, &cParams) );
+            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeContiguous);
+        }
+
+        DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
+        {   size_t cSizeIndep;
+            size_t const inSize = dictSize * 3;
+            size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
+            LZ4F_preferences_t cParams;
+            memset(&cParams, 0, sizeof(cParams));
+            cParams.frameInfo.blockMode = LZ4F_blockIndependent;
+            cParams.frameInfo.blockSizeID = LZ4F_max64KB;
+            CHECK_V(cSizeIndep,
+                LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
+                                              CNBuffer, inSize,
+                                              cdict, &cParams) );
+            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeIndep);
+        }
+
         LZ4F_freeCDict(cdict);
     }
 
-- 
cgit v0.12


From 757497ae3db93a78d146ff573ae267f54e49c9b6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 10 Aug 2017 16:53:57 -0700
Subject: implemented lz4frame decompression API

---
 doc/lz4frame_manual.html |  10 ++---
 lib/lz4frame.c           | 106 +++++++++++++++++++++++------------------------
 lib/lz4frame.h           |  10 ++---
 lib/lz4frame_static.h    |  20 ++++++---
 tests/frametest.c        |  67 ++++++++++++++++++++++++++++--
 5 files changed, 141 insertions(+), 72 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 9593181..2a625a6 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -176,13 +176,13 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 


 
 
size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
-
 To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
- It will flush whatever data remained within `cctx` (like LZ4_flush())
- and properly finalize the frame, with an endMark and a checksum.
+
  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+  It will flush whatever data remained within `cctx` (like LZ4_flush())
+  and properly finalize the frame, with an endMark and a checksum.
  `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
  @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
            or an error code if it fails (which can be tested using LZ4F_isError())
- A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+  A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  
 


 
@@ -190,7 +190,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 
typedef struct {
   unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
-  unsigned reserved[3];
+  unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 


 
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index fbccc9d..c27fc51 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -926,6 +926,8 @@ typedef enum {
 void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
 {
     dctx->dStage = dstage_getHeader;
+    dctx->dict = NULL;
+    dctx->dictSize = 0;
 }
 
 
@@ -1101,19 +1103,6 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
 }
 
 
-/* trivial redirector, for common prototype */
-static int LZ4F_decompress_safe (const char* src,
-                                 char* dst,
-                                 int compressedSize,
-                                 int dstCapacity,
-                                 const char* dictStart,
-                                 int dictSize)
-{
-    (void)dictStart; (void)dictSize;
-    return LZ4_decompress_safe (src, dst, compressedSize, dstCapacity);
-}
-
-
 static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
 {
     if (dctxPtr->dictSize==0)
@@ -1174,21 +1163,22 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi
 
 
 /*! LZ4F_decompress() :
- * Call this function repetitively to regenerate data compressed within srcBuffer.
- * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *  Call this function repetitively to regenerate compressed data in srcBuffer.
+ *  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
+ *  into dstBuffer of capacity *dstSizePtr.
  *
- * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *  The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
  *
- * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete.
- * Remaining data will have to be presented again in a subsequent invocation.
+ *  The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ *  If number of bytes read is < number of bytes provided, then decompression operation is not complete.
+ *  Remaining data will have to be presented again in a subsequent invocation.
  *
- * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
- * Basically, it's the size of the current (or remaining) compressed block + header of next block.
- * Respecting the hint provides some boost to performance, since it allows less buffer shuffling.
- * Note that this is just a hint, it's always possible to any srcSize value.
- * When a frame is fully decoded, @return will be 0.
- * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
+ *  The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
+ *  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ *  Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
+ *  Note that this is just a hint, and it's always possible to any srcSize value.
+ *  When a frame is fully decoded, @return will be 0.
+ *  If decompression failed, @return is an error code which can be tested using LZ4F_isError().
  */
 size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                        void* dstBuffer, size_t* dstSizePtr,
@@ -1265,8 +1255,6 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
             }   }
             dctxPtr->tmpInSize = 0;
             dctxPtr->tmpInTarget = 0;
-            dctxPtr->dict = dctxPtr->tmpOutBuffer;
-            dctxPtr->dictSize = 0;
             dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
             dctxPtr->tmpOutStart = 0;
             dctxPtr->tmpOutSize = 0;
@@ -1381,15 +1369,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
             break;
 
         case dstage_decodeCBlock_intoDst:
-            {   int (*decoder)(const char*, char*, int, int, const char*, int);
-                int decodedSize;
-
-                if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
-                    decoder = LZ4_decompress_safe_usingDict;
-                else
-                    decoder = LZ4F_decompress_safe;
-
-                decodedSize = decoder((const char*)selectedIn, (char*)dstPtr,
+            {   int const decodedSize = LZ4_decompress_safe_usingDict(
+                        (const char*)selectedIn, (char*)dstPtr,
                         (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
                         (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
                 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC);   /* decompression failed */
@@ -1409,13 +1390,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
 
         case dstage_decodeCBlock_intoTmp:
             /* not enough place into dst : decode into tmpOut */
-            {   int (*decoder)(const char*, char*, int, int, const char*, int);
-                int decodedSize;
-
-                if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked)
-                    decoder = LZ4_decompress_safe_usingDict;
-                else
-                    decoder = LZ4F_decompress_safe;
+            {   int decodedSize;
 
                 /* ensure enough place for tmpOut */
                 if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) {
@@ -1433,7 +1408,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 }
 
                 /* Decode */
-                decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut,
+                decodedSize = LZ4_decompress_safe_usingDict(
+                        (const char*)selectedIn, (char*)dctxPtr->tmpOut,
                         (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
                         (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
                 if (decodedSize < 0)
@@ -1476,7 +1452,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                     return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
                 if (suffixSize == 0) {  /* frame completed */
                     nextSrcSizeHint = 0;
-                    dctxPtr->dStage = dstage_getHeader;
+                    LZ4F_resetDecompressionContext(dctxPtr);
                     doAnotherStage = 0;
                     break;
                 }
@@ -1510,7 +1486,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh));
                 if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid);
                 nextSrcSizeHint = 0;
-                dctxPtr->dStage = dstage_getHeader;
+                LZ4F_resetDecompressionContext(dctxPtr);
                 doAnotherStage = 0;
                 break;
             }
@@ -1530,7 +1506,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
         case dstage_storeSFrameSize:
             {
                 size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
+                if (sizeToCopy > (size_t)(srcEnd - srcPtr))
+                    sizeToCopy = srcEnd - srcPtr;
                 memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
                 dctxPtr->tmpInSize += sizeToCopy;
@@ -1553,24 +1530,25 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
 
         case dstage_skipSkippable:
             {   size_t skipSize = dctxPtr->tmpInTarget;
-                if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr;
+                if (skipSize > (size_t)(srcEnd-srcPtr))
+                    skipSize = srcEnd-srcPtr;
                 srcPtr += skipSize;
                 dctxPtr->tmpInTarget -= skipSize;
                 doAnotherStage = 0;
                 nextSrcSizeHint = dctxPtr->tmpInTarget;
                 if (nextSrcSizeHint) break;
-                dctxPtr->dStage = dstage_getHeader;
+                LZ4F_resetDecompressionContext(dctxPtr);
                 break;
             }
         }
     }
 
-    /* preserve dictionary within tmp if necessary */
+    /* preserve history within tmp if necessary */
     if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
-        &&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
-        &&(!decompressOptionsPtr->stableDst)
-        &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1))
-        )
+      && (dctxPtr->dict != dctxPtr->tmpOutBuffer)
+      && (dctxPtr->dStage != dstage_getHeader)
+      && (!decompressOptionsPtr->stableDst)
+      && ((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
     {
         if (dctxPtr->dStage == dstage_flushOut) {
             size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
@@ -1600,3 +1578,23 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
     *dstSizePtr = (dstPtr - dstStart);
     return nextSrcSizeHint;
 }
+
+/*! LZ4F_decompress_usingDict() :
+ *  Same as LZ4F_decompress(), using a predefined dictionary.
+ *  Dictionary is used "in place", without any preprocessing.
+ *  It must remain accessible throughout the entire frame decoding.
+ */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+                       void* dstBuffer, size_t* dstSizePtr,
+                       const void* srcBuffer, size_t* srcSizePtr,
+                       const void* dict, size_t dictSize,
+                       const LZ4F_decompressOptions_t* decompressOptionsPtr)
+{
+    if (dctxPtr->dStage <= dstage_init) {
+        dctxPtr->dict = (const BYTE*)dict;
+        dctxPtr->dictSize = dictSize;
+    }
+    return LZ4F_decompress(dctxPtr, dstBuffer, dstSizePtr,
+                           srcBuffer, srcSizePtr,
+                           decompressOptionsPtr);
+}
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index dd76194..844255b 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -273,13 +273,13 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t
 LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
 
 /*! LZ4F_compressEnd() :
- * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
- * It will flush whatever data remained within `cctx` (like LZ4_flush())
- * and properly finalize the frame, with an endMark and a checksum.
+ *  To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
+ *  It will flush whatever data remained within `cctx` (like LZ4_flush())
+ *  and properly finalize the frame, with an endMark and a checksum.
  * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
  * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
  *           or an error code if it fails (which can be tested using LZ4F_isError())
- * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
+ *  A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
  */
 LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
 
@@ -292,7 +292,7 @@ typedef LZ4F_dctx* LZ4F_decompressionContext_t;   /* compatibility with previous
 
 typedef struct {
   unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
-  unsigned reserved[3];
+  unsigned reserved[3];  /* must be set to zero for forward compatibility */
 } LZ4F_decompressOptions_t;
 
 
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 5f22aed..b585b72 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -92,10 +92,11 @@ typedef struct LZ4F_CDict_s LZ4F_CDict;
  *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
  *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
  *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- *  `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
+ * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
 void        LZ4F_freeCDict(LZ4F_CDict* CDict);
 
+
 /*! LZ4_compressFrame_usingCDict() :
  *  Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary.
  *  If cdict==NULL, compress without a dictionary.
@@ -104,8 +105,7 @@ void        LZ4F_freeCDict(LZ4F_CDict* CDict);
  *  The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
  *  but it's not recommended, as it's the only way to provide dictID in the frame header.
  * @return : number of bytes written into dstBuffer.
- *           or an error code if it fails (can be tested using LZ4F_isError())
- */
+ *           or an error code if it fails (can be tested using LZ4F_isError()) */
 size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
                                const void* src, size_t srcSize,
                                const LZ4F_CDict* cdict,
@@ -118,14 +118,24 @@ size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
  * `prefsPtr` is optional : you may provide NULL as argument,
  *  however, it's the only way to provide dictID in the frame header.
  * @return : number of bytes written into dstBuffer for the header,
- *           or an error code (which can be tested using LZ4F_isError())
- */
+ *           or an error code (which can be tested using LZ4F_isError()) */
 size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
                                      void* dstBuffer, size_t dstCapacity,
                                      const LZ4F_CDict* cdict,
                                      const LZ4F_preferences_t* prefsPtr);
 
 
+/*! LZ4F_decompress_usingDict() :
+ *  Same as LZ4F_decompress(), using a predefined dictionary.
+ *  Dictionary is used "in place", without any preprocessing.
+ *  It must remain accessible throughout the entire frame decoding. */
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+                       void* dstBuffer, size_t* dstSizePtr,
+                       const void* srcBuffer, size_t* srcSizePtr,
+                       const void* dict, size_t dictSize,
+                       const LZ4F_decompressOptions_t* decompressOptionsPtr);
+
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/tests/frametest.c b/tests/frametest.c
index a30089f..d0665c5 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -505,8 +505,28 @@ int basicTests(U32 seed, double compressibility)
                 LZ4F_compressFrame_usingCDict(compressedBuffer, dstCapacity,
                                               CNBuffer, dictSize,
                                               cdict, NULL) );
-        DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeWithDict);
+        DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+                        (unsigned)dictSize, (unsigned)cSizeWithDict);
         if (cSizeWithDict >= cSizeNoDict) goto _output_error;  /* must be more efficient */
+        crcOrig = XXH64(CNBuffer, dictSize, 0);
+
+        DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
+        {   LZ4F_dctx* dctx;
+            size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+            size_t compressedSize = cSizeWithDict;
+            CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+            CHECK( LZ4F_decompress_usingDict(dctx,
+                                        decodedBuffer, &decodedSize,
+                                        compressedBuffer, &compressedSize,
+                                        CNBuffer, dictSize,
+                                        NULL) );
+            if (compressedSize != cSizeWithDict) goto _output_error;
+            if (decodedSize != dictSize) goto _output_error;
+            { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+              if (crcDest != crcOrig) goto _output_error; }
+            DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+            CHECK( LZ4F_freeDecompressionContext(dctx) );
+        }
 
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
         {   size_t cSizeLevelMax;
@@ -544,9 +564,30 @@ int basicTests(U32 seed, double compressibility)
                 LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
-            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeContiguous);
+            DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+                        (unsigned)inSize, (unsigned)cSizeContiguous);
+
+            DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
+            {   LZ4F_dctx* dctx;
+                size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+                size_t compressedSize = cSizeContiguous;
+                CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+                CHECK( LZ4F_decompress_usingDict(dctx,
+                                            decodedBuffer, &decodedSize,
+                                            compressedBuffer, &compressedSize,
+                                            CNBuffer, dictSize,
+                                            NULL) );
+                if (compressedSize != cSizeContiguous) goto _output_error;
+                if (decodedSize != inSize) goto _output_error;
+                crcOrig = XXH64(CNBuffer, inSize, 0);
+                { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+                  if (crcDest != crcOrig) goto _output_error; }
+                DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+                CHECK( LZ4F_freeDecompressionContext(dctx) );
+            }
         }
 
+
         DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
         {   size_t cSizeIndep;
             size_t const inSize = dictSize * 3;
@@ -559,7 +600,27 @@ int basicTests(U32 seed, double compressibility)
                 LZ4F_compressFrame_usingCDict(compressedBuffer, outCapacity,
                                               CNBuffer, inSize,
                                               cdict, &cParams) );
-            DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeIndep);
+            DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
+                        (unsigned)inSize, (unsigned)cSizeIndep);
+
+            DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
+            {   LZ4F_dctx* dctx;
+                size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+                size_t compressedSize = cSizeIndep;
+                CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+                CHECK( LZ4F_decompress_usingDict(dctx,
+                                            decodedBuffer, &decodedSize,
+                                            compressedBuffer, &compressedSize,
+                                            CNBuffer, dictSize,
+                                            NULL) );
+                if (compressedSize != cSizeIndep) goto _output_error;
+                if (decodedSize != inSize) goto _output_error;
+                crcOrig = XXH64(CNBuffer, inSize, 0);
+                { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
+                  if (crcDest != crcOrig) goto _output_error; }
+                DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+                CHECK( LZ4F_freeDecompressionContext(dctx) );
+            }
         }
 
         LZ4F_freeCDict(cdict);
-- 
cgit v0.12


From 77f99d2922a82bd3187dd7086a8b9dee06bbaed6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 11 Aug 2017 17:46:52 -0700
Subject: restored block checksum capability at lz4frame API level

---
 NEWS                     |   1 +
 doc/lz4frame_manual.html |  25 +-
 examples/frameCompress.c |   6 +-
 lib/lz4frame.c           | 699 ++++++++++++++++++++++++++---------------------
 lib/lz4frame.h           |  25 +-
 lib/lz4frame_static.h    |   2 +-
 tests/frametest.c        |  28 +-
 7 files changed, 445 insertions(+), 341 deletions(-)

diff --git a/NEWS b/NEWS
index d10c48f..63d53a6 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,7 @@ cli : added GNU separator -- specifying that all following arguments are files
 API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
 API : added LZ4F_resetDecompressionContext()
 API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
+API : lz4frame : can control block checksum and dictionary ID
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
 build : fix : static lib installation, by Ido Rosen
 build : dragonFlyBSD, OpenBSD, NetBSD supported
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 2a625a6..7529f6e 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -63,6 +63,11 @@
 } LZ4F_contentChecksum_t;
 


 
typedef enum {
+    LZ4F_noBlockChecksum=0,
+    LZ4F_blockChecksumEnabled
+} LZ4F_blockChecksum_t;
+


+
typedef enum {
     LZ4F_frame=0,
     LZ4F_skippableFrame
     LZ4F_OBSOLETE_ENUM(skippableFrame)
@@ -70,16 +75,16 @@
 


 
typedef struct {
   LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* blockLinked, blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
+  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
+  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
   unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
   unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  unsigned               reserved[1];          /* must be zero for forward compatibility */
+  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
 } LZ4F_frameInfo_t;
-
 makes it possible to supply detailed frame parameters to the stream interface.
- It's not required to set all fields, as long as the structure was initially memset() to zero.
- All reserved fields must be set to zero. 
+
  makes it possible to set or read frame parameters.
+  It's not required to set all fields, as long as the structure was initially memset() to zero.
+  For all fields, 0 sets it to default value 
 


 
 
typedef struct {
@@ -88,9 +93,9 @@
   unsigned autoFlush;              /* 1 == always flush, to reduce usage of internal buffers */
   unsigned reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
-
 makes it possible to supply detailed compression parameters to the stream interface.
- It's not required to set all fields, as long as the structure was initially memset() to zero.
- All reserved fields must be set to zero. 
+
  makes it possible to supply detailed compression parameters to the stream interface.
+  It's not required to set all fields, as long as the structure was initially memset() to zero.
+  All reserved fields must be set to zero. 
 


 
 
Simple compression function

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 0a42fe3..8712725 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -14,7 +14,7 @@
 
 static const LZ4F_preferences_t lz4_preferences = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
-      0 /* content size unknown */, 0 /* no dictID */ , { 0 } /* reserved */ },
+      0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
     0,   /* compression level */
     0,   /* autoflush */
     { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
@@ -266,7 +266,7 @@ int main(int argc, const char **argv) {
         ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
         if (ret) {
             printf("compress : failed with code %zu\n", ret);
-            return ret;
+            return (int)ret;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
             inpFilename, sizeIn, sizeOut,
@@ -286,7 +286,7 @@ int main(int argc, const char **argv) {
         ret = decompress_file(inpFp, outFp);
         if (ret) {
             printf("decompress : failed with code %zu\n", ret);
-            return ret;
+            return (int)ret;
         }
         printf("decompress : done\n");
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index c27fc51..e613901 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -68,7 +68,7 @@ You can contact the author at :
 
 
 /*-************************************
-*  Common Utils
+*  Debug
 **************************************/
 #define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
@@ -260,11 +260,11 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
     return requestedBSID;
 }
 
-/* LZ4F_compressBound_internal() :
- * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
- * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
- * Result is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
- * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
+/*! LZ4F_compressBound_internal() :
+ *  Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
+ *  prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
+ * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
+ *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  */
 static size_t LZ4F_compressBound_internal(size_t srcSize,
                                     const LZ4F_preferences_t* preferencesPtr,
@@ -275,8 +275,8 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
     prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;   /* worst case */
     {   const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
         U32 const flush = prefsPtr->autoFlush | (srcSize==0);
-        LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID;
-        size_t const blockSize = LZ4F_getBlockSize(bid);
+        LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
+        size_t const blockSize = LZ4F_getBlockSize(blockID);
         size_t const maxBuffered = blockSize - 1;
         size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
         size_t const maxSrcSize = srcSize + bufferedSize;
@@ -285,10 +285,12 @@ static size_t LZ4F_compressBound_internal(size_t srcSize,
         size_t const lastBlockSize = flush ? partialBlockSize : 0;
         unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
 
-        size_t const blockHeaderSize = 4;   /* default, without block CRC option (which cannot be generated with current API) */
+        size_t const blockHeaderSize = 4;
+        size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
         size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
 
-        return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;;
+        return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
+               (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
     }
 }
 
@@ -400,11 +402,11 @@ struct LZ4F_CDict_s {
     LZ4_streamHC_t* HCCtx;
 }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
 
-/*! LZ4_createCDict() :
+/*! LZ4F_createCDict() :
  *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
- *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
- *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict
+ *  LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
  * @return : digested dictionary for compression, or NULL if failed */
 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
 {
@@ -559,10 +561,11 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
 
     /* FLG Byte */
     *dstPtr++ = (BYTE)(((1 & _2BITS) << 6)    /* Version('01') */
-        + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)  /* Block mode */
-        + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)     /* Frame content size */
-        + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)   /* Frame checksum */
-        +  (cctxPtr->prefs.frameInfo.dictID > 0) );   /* Dictionary ID */
+        + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
+        + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
+        + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
+        + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
+        +  (cctxPtr->prefs.frameInfo.dictID > 0) );
     /* BD Byte */
     *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
     /* Optional Frame content size field */
@@ -614,9 +617,14 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP
 
 typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
 
-static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict)
+
+/*! LZ4F_makeBlock():
+ *  compress a single block, add header and checksum
+ *  assumption : dst buffer capacity is >= srcSize */
+static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
+                             compressFunc_t compress, void* lz4ctx, int level,
+                             const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
 {
-    /* compress a single block */
     BYTE* const cSizePtr = (BYTE*)dst;
     U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
                                       (int)(srcSize), (int)(srcSize-1),
@@ -627,7 +635,11 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compres
         LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
         memcpy(cSizePtr+4, src, srcSize);
     }
-    return cSize + 4;
+    if (crcFlag) {
+        U32 const crc32 = XXH32(cSizePtr+4, cSize, 0);  /* checksum of compressed data */
+        LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
+    }
+    return 4 + cSize + ((U32)crcFlag)*4;
 }
 
 
@@ -690,7 +702,10 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
  * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  */
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+                           void* dstBuffer, size_t dstCapacity,
+                     const void* srcBuffer, size_t srcSize,
+                     const LZ4F_compressOptions_t* compressOptionsPtr)
 {
     LZ4F_compressOptions_t cOptionsNull;
     size_t const blockSize = cctxPtr->maxBlockSize;
@@ -722,7 +737,9 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
             memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
             srcPtr += sizeToCopy;
 
-            dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
+            dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
+                                     compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                                     cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
 
             if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
             cctxPtr->tmpInSize = 0;
@@ -732,14 +749,18 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapaci
     while ((size_t)(srcEnd - srcPtr) >= blockSize) {
         /* compress full blocks */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
+                                 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                                 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
         srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
         /* compress remaining input < blockSize */
         lastBlockCompressed = fromSrcBuffer;
-        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
+        dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
+                                 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                                 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
         srcPtr  = srcEnd;
     }
 
@@ -801,7 +822,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const
     compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
 
     /* compress tmp buffer */
-    dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict);
+    dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
+                             compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                             cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
     if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
     cctxPtr->tmpInSize = 0;
 
@@ -858,10 +881,23 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize,
 *   Frame Decompression
 *****************************************************/
 
+typedef enum {
+    dstage_getFrameHeader=0, dstage_storeFrameHeader,
+    dstage_init,
+    dstage_getBlockHeader, dstage_storeBlockHeader,
+    dstage_copyDirect, dstage_getBlockChecksum,
+    dstage_getCBlock, dstage_storeCBlock,
+    dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
+    dstage_decodeCBlock_intoTmp, dstage_flushOut,
+    dstage_getSuffix, dstage_storeSuffix,
+    dstage_getSFrameSize, dstage_storeSFrameSize,
+    dstage_skipSkippable
+} dStage_t;
+
 struct LZ4F_dctx_s {
     LZ4F_frameInfo_t frameInfo;
     U32    version;
-    U32    dStage;
+    dStage_t dStage;
     U64    frameRemainingSize;
     size_t maxBlockSize;
     size_t maxBufferSize;
@@ -869,40 +905,41 @@ struct LZ4F_dctx_s {
     size_t tmpInSize;
     size_t tmpInTarget;
     BYTE*  tmpOutBuffer;
-    const BYTE*  dict;
+    const BYTE* dict;
     size_t dictSize;
     BYTE*  tmpOut;
     size_t tmpOutSize;
     size_t tmpOutStart;
     XXH32_state_t xxh;
+    XXH32_state_t blockChecksum;
     BYTE   header[16];
 };  /* typedef'd to LZ4F_dctx in lz4frame.h */
 
 
 /*! LZ4F_createDecompressionContext() :
- *   Create a decompressionContext object, which will track all decompression operations.
- *   Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
- *   Object can later be released using LZ4F_freeDecompressionContext().
- *   @return : if != 0, there was an error during context creation.
+ *  Create a decompressionContext object, which will track all decompression operations.
+ *  Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
+ *  Object can later be released using LZ4F_freeDecompressionContext().
+ * @return : if != 0, there was an error during context creation.
  */
 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
-    LZ4F_dctx* const dctxPtr = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
-    if (dctxPtr==NULL) return err0r(LZ4F_ERROR_GENERIC);
+    LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
+    if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
 
-    dctxPtr->version = versionNumber;
-    *LZ4F_decompressionContextPtr = dctxPtr;
+    dctx->version = versionNumber;
+    *LZ4F_decompressionContextPtr = dctx;
     return LZ4F_OK_NoError;
 }
 
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr)
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
 {
     LZ4F_errorCode_t result = LZ4F_OK_NoError;
-    if (dctxPtr != NULL) {   /* can accept NULL input, like free() */
-      result = (LZ4F_errorCode_t)dctxPtr->dStage;
-      FREEMEM(dctxPtr->tmpIn);
-      FREEMEM(dctxPtr->tmpOutBuffer);
-      FREEMEM(dctxPtr);
+    if (dctx != NULL) {   /* can accept NULL input, like free() */
+      result = (LZ4F_errorCode_t)dctx->dStage;
+      FREEMEM(dctx->tmpIn);
+      FREEMEM(dctx->tmpOutBuffer);
+      FREEMEM(dctx);
     }
     return result;
 }
@@ -910,22 +947,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctxPtr)
 
 /*==---   Streaming Decompression operations   ---==*/
 
-typedef enum {
-    dstage_getHeader=0, dstage_storeHeader,
-    dstage_init,
-    dstage_getCBlockSize, dstage_storeCBlockSize,
-    dstage_copyDirect,
-    dstage_getCBlock, dstage_storeCBlock,
-    dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
-    dstage_decodeCBlock_intoTmp, dstage_flushOut,
-    dstage_getSuffix, dstage_storeSuffix,
-    dstage_getSFrameSize, dstage_storeSFrameSize,
-    dstage_skipSkippable
-} dStage_t;
-
 void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
 {
-    dctx->dStage = dstage_getHeader;
+    dctx->dStage = dstage_getFrameHeader;
     dctx->dict = NULL;
     dctx->dictSize = 0;
 }
@@ -959,31 +983,31 @@ static size_t LZ4F_headerSize(const void* src, size_t srcSize)
 /*! LZ4F_decodeHeader() :
  *  input   : `src` points at the **beginning of the frame**
  *  output  : set internal values of dctx, such as
- *            dctxPtr->frameInfo and dctxPtr->dStage.
+ *            dctx->frameInfo and dctx->dStage.
  *            Also allocates internal buffers.
  *  @return : nb Bytes read from src (necessarily <= srcSize)
  *            or an error code (testable with LZ4F_isError())
  */
-static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcSize)
+static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
 {
-    unsigned blockMode, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
+    unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
     size_t frameHeaderSize;
     const BYTE* srcPtr = (const BYTE*)src;
 
     /* need to decode header to get frameInfo */
     if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete);   /* minimal frame header size */
-    memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
+    memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
 
     /* special case : skippable frames */
     if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
-        dctxPtr->frameInfo.frameType = LZ4F_skippableFrame;
-        if (src == (void*)(dctxPtr->header)) {
-            dctxPtr->tmpInSize = srcSize;
-            dctxPtr->tmpInTarget = 8;
-            dctxPtr->dStage = dstage_storeSFrameSize;
+        dctx->frameInfo.frameType = LZ4F_skippableFrame;
+        if (src == (void*)(dctx->header)) {
+            dctx->tmpInSize = srcSize;
+            dctx->tmpInTarget = 8;
+            dctx->dStage = dstage_storeSFrameSize;
             return srcSize;
         } else {
-            dctxPtr->dStage = dstage_getSFrameSize;
+            dctx->dStage = dstage_getSFrameSize;
             return 4;
         }
     }
@@ -991,20 +1015,19 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     /* control magic number */
     if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
         return err0r(LZ4F_ERROR_frameType_unknown);
-    dctxPtr->frameInfo.frameType = LZ4F_frame;
+    dctx->frameInfo.frameType = LZ4F_frame;
 
     /* Flags */
     {   U32 const FLG = srcPtr[4];
         U32 const version = (FLG>>6) & _2BITS;
-        U32 const blockChecksumFlag = (FLG>>4) & _1BIT;
+        blockChecksumFlag = (FLG>>4) & _1BIT;
         blockMode = (FLG>>5) & _1BIT;
         contentSizeFlag = (FLG>>3) & _1BIT;
         contentChecksumFlag = (FLG>>2) & _1BIT;
         dictIDFlag = FLG & _1BIT;
         /* validate */
-        if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
+        if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
         if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong);        /* Version Number, only supported value */
-        if (blockChecksumFlag != 0) return err0r(LZ4F_ERROR_blockChecksum_unsupported); /* Not supported for the time being */
     }
 
     /* Frame Header Size */
@@ -1012,11 +1035,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
 
     if (srcSize < frameHeaderSize) {
         /* not enough input to fully decode frame header */
-        if (srcPtr != dctxPtr->header)
-            memcpy(dctxPtr->header, srcPtr, srcSize);
-        dctxPtr->tmpInSize = srcSize;
-        dctxPtr->tmpInTarget = frameHeaderSize;
-        dctxPtr->dStage = dstage_storeHeader;
+        if (srcPtr != dctx->header)
+            memcpy(dctx->header, srcPtr, srcSize);
+        dctx->tmpInSize = srcSize;
+        dctx->tmpInTarget = frameHeaderSize;
+        dctx->dStage = dstage_storeFrameHeader;
         return srcSize;
     }
 
@@ -1035,49 +1058,50 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctxPtr, const void* src, size_t srcS
     }
 
     /* save */
-    dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
-    dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
-    dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
-    dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+    dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
+    dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
+    dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
+    dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
+    dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
     if (contentSizeFlag)
-        dctxPtr->frameRemainingSize =
-            dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
+        dctx->frameRemainingSize =
+            dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
     if (dictIDFlag)
-        dctxPtr->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
+        dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
 
-    dctxPtr->dStage = dstage_init;
+    dctx->dStage = dstage_init;
 
     return frameHeaderSize;
 }
 
 
 /*! LZ4F_getFrameInfo() :
- * This function extracts frame parameters (max blockSize, frame checksum, etc.).
- * Usage is optional. Objective is to provide relevant information for allocation purposes.
- * This function works in 2 situations :
+ *  This function extracts frame parameters (max blockSize, frame checksum, etc.).
+ *  Usage is optional. Objective is to provide relevant information for allocation purposes.
+ *  This function works in 2 situations :
  *   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
  *     Amount of input data provided must be large enough to successfully decode the frame header.
  *     A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
  *   - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
- * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
- * Decompression must resume from (srcBuffer + *srcSizePtr).
+ *  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ *  Decompression must resume from (srcBuffer + *srcSizePtr).
  * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
  *           or an error code which can be tested using LZ4F_isError()
- * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
- * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
+ *  note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ *  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  */
-LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameInfoPtr,
+LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
                                    const void* srcBuffer, size_t* srcSizePtr)
 {
-    if (dctxPtr->dStage > dstage_storeHeader) {  /* assumption :  dstage_* header enum at beginning of range */
+    if (dctx->dStage > dstage_storeFrameHeader) {  /* assumption :  dstage_* header enum at beginning of range */
         /* frameInfo already decoded */
         size_t o=0, i=0;
         *srcSizePtr = 0;
-        *frameInfoPtr = dctxPtr->frameInfo;
+        *frameInfoPtr = dctx->frameInfo;
         /* returns : recommended nb of bytes for LZ4F_decompress() */
-        return LZ4F_decompress(dctxPtr, NULL, &o, NULL, &i, NULL);
+        return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
     } else {
-        if (dctxPtr->dStage == dstage_storeHeader) {
+        if (dctx->dStage == dstage_storeFrameHeader) {
             /* frame decoding already started, in the middle of header => automatic fail */
             *srcSizePtr = 0;
             return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
@@ -1090,73 +1114,75 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctxPtr, LZ4F_frameInfo_t* frameIn
                 return err0r(LZ4F_ERROR_frameHeader_incomplete);
             }
 
-            decodeResult = LZ4F_decodeHeader(dctxPtr, srcBuffer, hSize);
+            decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
             if (LZ4F_isError(decodeResult)) {
                 *srcSizePtr = 0;
             } else {
                 *srcSizePtr = decodeResult;
                 decodeResult = BHSize;   /* block header size */
             }
-            *frameInfoPtr = dctxPtr->frameInfo;
+            *frameInfoPtr = dctx->frameInfo;
             return decodeResult;
     }   }
 }
 
 
-static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
+/* LZ4F_updateDict() :
+ * only used for LZ4F_blockLinked mode */
+static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
 {
-    if (dctxPtr->dictSize==0)
-        dctxPtr->dict = (const BYTE*)dstPtr;   /* priority to dictionary continuity */
+    if (dctx->dictSize==0)
+        dctx->dict = (const BYTE*)dstPtr;   /* priority to dictionary continuity */
 
-    if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) {  /* dictionary continuity */
-        dctxPtr->dictSize += dstSize;
+    if (dctx->dict + dctx->dictSize == dstPtr) {  /* dictionary continuity */
+        dctx->dictSize += dstSize;
         return;
     }
 
     if (dstPtr - dstPtr0 + dstSize >= 64 KB) {  /* dstBuffer large enough to become dictionary */
-        dctxPtr->dict = (const BYTE*)dstPtr0;
-        dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize;
+        dctx->dict = (const BYTE*)dstPtr0;
+        dctx->dictSize = dstPtr - dstPtr0 + dstSize;
         return;
     }
 
-    if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) {
-        /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */
-        dctxPtr->dictSize += dstSize;
+    if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
+        /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
+        dctx->dictSize += dstSize;
         return;
     }
 
     if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
-        size_t const preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
-        size_t copySize = 64 KB - dctxPtr->tmpOutSize;
-        const BYTE* const oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
-        if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+        size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+        size_t copySize = 64 KB - dctx->tmpOutSize;
+        const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+        if (dctx->tmpOutSize > 64 KB) copySize = 0;
         if (copySize > preserveSize) copySize = preserveSize;
 
-        memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+        memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
 
-        dctxPtr->dict = dctxPtr->tmpOutBuffer;
-        dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize;
+        dctx->dict = dctx->tmpOutBuffer;
+        dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
         return;
     }
 
-    if (dctxPtr->dict == dctxPtr->tmpOutBuffer) {    /* copy dst into tmp to complete dict */
-        if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) {  /* tmp buffer not large enough */
+    if (dctx->dict == dctx->tmpOutBuffer) {    /* copy dst into tmp to complete dict */
+        if (dctx->dictSize + dstSize > dctx->maxBufferSize) {  /* tmp buffer not large enough */
             size_t const preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
-            memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
-            dctxPtr->dictSize = preserveSize;
+            memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+            dctx->dictSize = preserveSize;
         }
-        memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize);
-        dctxPtr->dictSize += dstSize;
+        memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
+        dctx->dictSize += dstSize;
         return;
     }
 
     /* join dict & dest into tmp */
     {   size_t preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
-        if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize;
-        memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
-        memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize);
-        dctxPtr->dict = dctxPtr->tmpOutBuffer;
-        dctxPtr->dictSize = preserveSize + dstSize;
+        if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
+        memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
+        memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
+        dctx->dict = dctx->tmpOutBuffer;
+        dctx->dictSize = preserveSize + dstSize;
     }
 }
 
@@ -1180,7 +1206,7 @@ static void LZ4F_updateDict(LZ4F_dctx* dctxPtr, const BYTE* dstPtr, size_t dstSi
  *  When a frame is fully decoded, @return will be 0.
  *  If decompression failed, @return is an error code which can be tested using LZ4F_isError().
  */
-size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
+size_t LZ4F_decompress(LZ4F_dctx* dctx,
                        void* dstBuffer, size_t* dstSizePtr,
                        const void* srcBuffer, size_t* srcSizePtr,
                        const LZ4F_decompressOptions_t* decompressOptionsPtr)
@@ -1202,243 +1228,288 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
     *srcSizePtr = 0;
     *dstSizePtr = 0;
 
-    /* behaves like a state machine */
+    /* behaves as a state machine */
 
     while (doAnotherStage) {
 
-        switch(dctxPtr->dStage)
+        switch(dctx->dStage)
         {
 
-        case dstage_getHeader:
+        case dstage_getFrameHeader:
             if ((size_t)(srcEnd-srcPtr) >= maxFHSize) {  /* enough to decode - shortcut */
-                LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);  /* will change dStage appropriately */
+                size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr);  /* will update dStage appropriately */
                 if (LZ4F_isError(hSize)) return hSize;
                 srcPtr += hSize;
                 break;
             }
-            dctxPtr->tmpInSize = 0;
+            dctx->tmpInSize = 0;
             if (srcEnd-srcPtr == 0) return minFHSize;   /* 0-size input */
-            dctxPtr->tmpInTarget = minFHSize;   /* minimum to attempt decode */
-            dctxPtr->dStage = dstage_storeHeader;
+            dctx->tmpInTarget = minFHSize;   /* minimum to attempt decode */
+            dctx->dStage = dstage_storeFrameHeader;
             /* fall-through */
 
-        case dstage_storeHeader:
-            {   size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy =  srcEnd - srcPtr;
-                memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
-                dctxPtr->tmpInSize += sizeToCopy;
+        case dstage_storeFrameHeader:
+            {   size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
+                memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+                dctx->tmpInSize += sizeToCopy;
                 srcPtr += sizeToCopy;
-                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
-                    nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize;   /* rest of header + nextBlockHeader */
+                if (dctx->tmpInSize < dctx->tmpInTarget) {
+                    nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;   /* rest of header + nextBlockHeader */
                     doAnotherStage = 0;   /* not enough src data, ask for some more */
                     break;
                 }
-                {   LZ4F_errorCode_t const hSize = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);  /* will change dStage appropriately */
+                {   size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget);  /* will update dStage appropriately */
                     if (LZ4F_isError(hSize)) return hSize;
                 }
                 break;
             }
 
         case dstage_init:
-            if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
+            if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
             /* internal buffers allocation */
-            {   size_t const bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
-                if (bufferNeeded > dctxPtr->maxBufferSize) {   /* tmp buffers too small */
-                    dctxPtr->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
-                    FREEMEM(dctxPtr->tmpIn);
-                    dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
-                    if (dctxPtr->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-                    FREEMEM(dctxPtr->tmpOutBuffer);
-                    dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
-                    if (dctxPtr->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
-                    dctxPtr->maxBufferSize = bufferNeeded;
+            {   size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */;
+                if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
+                    dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
+                    FREEMEM(dctx->tmpIn);
+                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
+                    if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    FREEMEM(dctx->tmpOutBuffer);
+                    dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
+                    if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    dctx->maxBufferSize = bufferNeeded;
             }   }
-            dctxPtr->tmpInSize = 0;
-            dctxPtr->tmpInTarget = 0;
-            dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
-            dctxPtr->tmpOutStart = 0;
-            dctxPtr->tmpOutSize = 0;
+            dctx->tmpInSize = 0;
+            dctx->tmpInTarget = 0;
+            dctx->tmpOut = dctx->tmpOutBuffer;
+            dctx->tmpOutStart = 0;
+            dctx->tmpOutSize = 0;
 
-            dctxPtr->dStage = dstage_getCBlockSize;
+            dctx->dStage = dstage_getBlockHeader;
             /* fall-through */
 
-        case dstage_getCBlockSize:
+        case dstage_getBlockHeader:
             if ((size_t)(srcEnd - srcPtr) >= BHSize) {
                 selectedIn = srcPtr;
                 srcPtr += BHSize;
             } else {
                 /* not enough input to read cBlockSize field */
-                dctxPtr->tmpInSize = 0;
-                dctxPtr->dStage = dstage_storeCBlockSize;
+                dctx->tmpInSize = 0;
+                dctx->dStage = dstage_storeBlockHeader;
             }
 
-            if (dctxPtr->dStage == dstage_storeCBlockSize)   /* can be skipped */
-        case dstage_storeCBlockSize:
-            {   size_t sizeToCopy = BHSize - dctxPtr->tmpInSize;
+            if (dctx->dStage == dstage_storeBlockHeader)   /* can be skipped */
+        case dstage_storeBlockHeader:
+            {   size_t sizeToCopy = BHSize - dctx->tmpInSize;
                 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
-                memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+                memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
-                dctxPtr->tmpInSize += sizeToCopy;
-                if (dctxPtr->tmpInSize < BHSize) {   /* not enough input for cBlockSize */
-                    nextSrcSizeHint = BHSize - dctxPtr->tmpInSize;
+                dctx->tmpInSize += sizeToCopy;
+                if (dctx->tmpInSize < BHSize) {   /* not enough input for cBlockSize */
+                    nextSrcSizeHint = BHSize - dctx->tmpInSize;
                     doAnotherStage  = 0;
                     break;
                 }
-                selectedIn = dctxPtr->tmpIn;
+                selectedIn = dctx->tmpIn;
             }
 
-        /* case dstage_decodeCBlockSize: */   /* no more direct access, to remove scan-build warning */
+        /* decode block header */
             {   size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
-                if (nextCBlockSize==0) {  /* frameEnd signal, no more CBlock */
-                    dctxPtr->dStage = dstage_getSuffix;
+                size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
+                if (nextCBlockSize==0) {  /* frameEnd signal, no more block */
+                    dctx->dStage = dstage_getSuffix;
                     break;
                 }
-                if (nextCBlockSize > dctxPtr->maxBlockSize)
+                if (nextCBlockSize > dctx->maxBlockSize)
                     return err0r(LZ4F_ERROR_maxBlockSize_invalid);
-                dctxPtr->tmpInTarget = nextCBlockSize;
                 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
-                    dctxPtr->dStage = dstage_copyDirect;
+                    /* next block is uncompressed */
+                    dctx->tmpInTarget = nextCBlockSize;
+                    if (dctx->frameInfo.blockChecksumFlag) {
+                        XXH32_reset(&dctx->blockChecksum, 0);
+                    }
+                    dctx->dStage = dstage_copyDirect;
                     break;
                 }
-                dctxPtr->dStage = dstage_getCBlock;
+                /* next block is a compressed block */
+                dctx->tmpInTarget = nextCBlockSize + crcSize;
+                dctx->dStage = dstage_getCBlock;
                 if (dstPtr==dstEnd) {
-                    nextSrcSizeHint = nextCBlockSize + BHSize;
+                    nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
                     doAnotherStage = 0;
                 }
                 break;
             }
 
         case dstage_copyDirect:   /* uncompressed block */
-            {   size_t sizeToCopy = dctxPtr->tmpInTarget;
-                if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr;
-                if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
+            {   size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
+                size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
                 memcpy(dstPtr, srcPtr, sizeToCopy);
-                if (dctxPtr->frameInfo.contentChecksumFlag)
-                    XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
-                if (dctxPtr->frameInfo.contentSize)
-                    dctxPtr->frameRemainingSize -= sizeToCopy;
+                if (dctx->frameInfo.blockChecksumFlag) {
+                    XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
+                }
+                if (dctx->frameInfo.contentChecksumFlag)
+                    XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
+                if (dctx->frameInfo.contentSize)
+                    dctx->frameRemainingSize -= sizeToCopy;
 
-                /* dictionary management */
-                if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
-                    LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0);
+                /* history management (linked blocks only)*/
+                if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
+                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
 
                 srcPtr += sizeToCopy;
                 dstPtr += sizeToCopy;
-                if (sizeToCopy == dctxPtr->tmpInTarget) {  /* all copied */
-                    dctxPtr->dStage = dstage_getCBlockSize;
+                if (sizeToCopy == dctx->tmpInTarget) {   /* all done */
+                    if (dctx->frameInfo.blockChecksumFlag) {
+                        dctx->tmpInSize = 0;
+                        dctx->dStage = dstage_getBlockChecksum;
+                    } else
+                        dctx->dStage = dstage_getBlockHeader;  /* new block */
                     break;
                 }
-                dctxPtr->tmpInTarget -= sizeToCopy;   /* still need to copy more */
-                nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize;
+                dctx->tmpInTarget -= sizeToCopy;  /* need to copy more */
+                nextSrcSizeHint = dctx->tmpInTarget +
+                                + dctx->frameInfo.contentChecksumFlag * 4  /* block checksum */
+                                + BHSize /* next header size */;
                 doAnotherStage = 0;
                 break;
             }
 
+        /* check block checksum for recently transferred uncompressed block */
+        case dstage_getBlockChecksum:
+            {   const void* crcSrc;
+                if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
+                    crcSrc = srcPtr;
+                    srcPtr += 4;
+                } else {
+                    size_t const stillToCopy = 4 - dctx->tmpInSize;
+                    size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
+                    memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
+                    dctx->tmpInSize += sizeToCopy;
+                    srcPtr += sizeToCopy;
+                    if (dctx->tmpInSize < 4) {  /* all input consumed */
+                        doAnotherStage = 0;
+                        break;
+                    }
+                    crcSrc = dctx->header;
+                }
+                {   U32 const readCRC = LZ4F_readLE32(crcSrc);
+                    U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
+                    if (readCRC != calcCRC)
+                        return err0r(LZ4F_ERROR_blockChecksum_invalid);
+                }
+            }
+            dctx->dStage = dstage_getBlockHeader;  /* new block */
+            break;
+
         case dstage_getCBlock:   /* entry from dstage_decodeCBlockSize */
-            if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) {
-                dctxPtr->tmpInSize = 0;
-                dctxPtr->dStage = dstage_storeCBlock;
+            if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
+                dctx->tmpInSize = 0;
+                dctx->dStage = dstage_storeCBlock;
                 break;
             }
+            /* input large enough to read full block directly */
             selectedIn = srcPtr;
-            srcPtr += dctxPtr->tmpInTarget;
-            dctxPtr->dStage = dstage_decodeCBlock;
+            srcPtr += dctx->tmpInTarget;
+            dctx->dStage = dstage_decodeCBlock;
             break;
 
         case dstage_storeCBlock:
-            {   size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr;
-                memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
-                dctxPtr->tmpInSize += sizeToCopy;
+            {   size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr));
+                memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
+                dctx->tmpInSize += sizeToCopy;
                 srcPtr += sizeToCopy;
-                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */
-                    nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize;
+                if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
+                    nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
                     doAnotherStage=0;
                     break;
                 }
-                selectedIn = dctxPtr->tmpIn;
-                dctxPtr->dStage = dstage_decodeCBlock;
+                selectedIn = dctx->tmpIn;
+                dctx->dStage = dstage_decodeCBlock;
             }
             /* fall-through */
 
+        /* At this stage, input is large enough to decode a block */
         case dstage_decodeCBlock:
-            if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize)   /* not enough place into dst : decode into tmpOut */
-                dctxPtr->dStage = dstage_decodeCBlock_intoTmp;
+            if (dctx->frameInfo.blockChecksumFlag) {
+                dctx->tmpInTarget -= 4;
+                {   U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
+                    U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
+                    if (readBlockCrc != calcBlockCrc)
+                        return err0r(LZ4F_ERROR_blockChecksum_invalid);
+            }   }
+            if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize)   /* not enough place into dst : decode into tmpOut */
+                dctx->dStage = dstage_decodeCBlock_intoTmp;
             else
-                dctxPtr->dStage = dstage_decodeCBlock_intoDst;
+                dctx->dStage = dstage_decodeCBlock_intoDst;
             break;
 
         case dstage_decodeCBlock_intoDst:
             {   int const decodedSize = LZ4_decompress_safe_usingDict(
                         (const char*)selectedIn, (char*)dstPtr,
-                        (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
-                        (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+                        (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+                        (const char*)dctx->dict, (int)dctx->dictSize);
                 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC);   /* decompression failed */
-                if (dctxPtr->frameInfo.contentChecksumFlag)
-                    XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
-                if (dctxPtr->frameInfo.contentSize)
-                    dctxPtr->frameRemainingSize -= decodedSize;
+                if (dctx->frameInfo.contentChecksumFlag)
+                    XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
+                if (dctx->frameInfo.contentSize)
+                    dctx->frameRemainingSize -= decodedSize;
 
                 /* dictionary management */
-                if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
-                    LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0);
+                if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+                    LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
 
                 dstPtr += decodedSize;
-                dctxPtr->dStage = dstage_getCBlockSize;
+                dctx->dStage = dstage_getBlockHeader;
                 break;
             }
 
         case dstage_decodeCBlock_intoTmp:
             /* not enough place into dst : decode into tmpOut */
-            {   int decodedSize;
-
-                /* ensure enough place for tmpOut */
-                if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) {
-                    if (dctxPtr->dict == dctxPtr->tmpOutBuffer) {
-                        if (dctxPtr->dictSize > 128 KB) {
-                            memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
-                            dctxPtr->dictSize = 64 KB;
-                        }
-                        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
-                    } else {  /* dict not within tmp */
-                        size_t reservedDictSpace = dctxPtr->dictSize;
-                        if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB;
-                        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace;
+
+            /* ensure enough place for tmpOut */
+            if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
+                if (dctx->dict == dctx->tmpOutBuffer) {
+                    if (dctx->dictSize > 128 KB) {
+                        memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
+                        dctx->dictSize = 64 KB;
                     }
+                    dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
+                } else {  /* dict not within tmp */
+                    size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
+                    dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
                 }
+            }
 
-                /* Decode */
-                decodedSize = LZ4_decompress_safe_usingDict(
-                        (const char*)selectedIn, (char*)dctxPtr->tmpOut,
-                        (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize,
-                        (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
-                if (decodedSize < 0)
-                    return err0r(LZ4F_ERROR_decompressionFailed);   /* decompression failed */
-                if (dctxPtr->frameInfo.contentChecksumFlag)
-                    XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
-                if (dctxPtr->frameInfo.contentSize)
-                    dctxPtr->frameRemainingSize -= decodedSize;
-                dctxPtr->tmpOutSize = decodedSize;
-                dctxPtr->tmpOutStart = 0;
-                dctxPtr->dStage = dstage_flushOut;
-                break;
+            /* Decode block */
+            {   int const decodedSize = LZ4_decompress_safe_usingDict(
+                        (const char*)selectedIn, (char*)dctx->tmpOut,
+                        (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
+                        (const char*)dctx->dict, (int)dctx->dictSize);
+                if (decodedSize < 0)  /* decompression failed */
+                    return err0r(LZ4F_ERROR_decompressionFailed);
+                if (dctx->frameInfo.contentChecksumFlag)
+                    XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
+                if (dctx->frameInfo.contentSize)
+                    dctx->frameRemainingSize -= decodedSize;
+                dctx->tmpOutSize = decodedSize;
+                dctx->tmpOutStart = 0;
+                dctx->dStage = dstage_flushOut;
             }
+            /* fall-through */
 
         case dstage_flushOut:  /* flush decoded data from tmpOut to dstBuffer */
-            {   size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart;
-                if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr;
-                memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy);
+            {   size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
+                memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
 
                 /* dictionary management */
-                if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
-                    LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1);
+                if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+                    LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
 
-                dctxPtr->tmpOutStart += sizeToCopy;
+                dctx->tmpOutStart += sizeToCopy;
                 dstPtr += sizeToCopy;
 
-                /* end of flush ? */
-                if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) {
-                    dctxPtr->dStage = dstage_getCBlockSize;
+                if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
+                    dctx->dStage = dstage_getBlockHeader;  /* get next block */
                     break;
                 }
                 nextSrcSizeHint = BHSize;
@@ -1447,46 +1518,47 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
             }
 
         case dstage_getSuffix:
-            {   size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
-                if (dctxPtr->frameRemainingSize)
+            {   size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4;
+                if (dctx->frameRemainingSize)
                     return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
                 if (suffixSize == 0) {  /* frame completed */
                     nextSrcSizeHint = 0;
-                    LZ4F_resetDecompressionContext(dctxPtr);
+                    LZ4F_resetDecompressionContext(dctx);
                     doAnotherStage = 0;
                     break;
                 }
                 if ((srcEnd - srcPtr) < 4) {  /* not enough size for entire CRC */
-                    dctxPtr->tmpInSize = 0;
-                    dctxPtr->dStage = dstage_storeSuffix;
+                    dctx->tmpInSize = 0;
+                    dctx->dStage = dstage_storeSuffix;
                 } else {
                     selectedIn = srcPtr;
                     srcPtr += 4;
                 }
             }
 
-            if (dctxPtr->dStage == dstage_storeSuffix)   /* can be skipped */
+            if (dctx->dStage == dstage_storeSuffix)   /* can be skipped */
         case dstage_storeSuffix:
             {
-                size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
+                size_t sizeToCopy = 4 - dctx->tmpInSize;
                 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
-                memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+                memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
-                dctxPtr->tmpInSize += sizeToCopy;
-                if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */
-                    nextSrcSizeHint = 4 - dctxPtr->tmpInSize;
+                dctx->tmpInSize += sizeToCopy;
+                if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
+                    nextSrcSizeHint = 4 - dctx->tmpInSize;
                     doAnotherStage=0;
                     break;
                 }
-                selectedIn = dctxPtr->tmpIn;
+                selectedIn = dctx->tmpIn;
             }
 
-        /* case dstage_checkSuffix: */   /* no direct call, to avoid scan-build warning */
+        /* case dstage_checkSuffix: */   /* no direct call, avoid scan-build warning */
             {   U32 const readCRC = LZ4F_readLE32(selectedIn);
-                U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh));
-                if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid);
+                U32 const resultCRC = XXH32_digest(&(dctx->xxh));
+                if (readCRC != resultCRC)
+                    return err0r(LZ4F_ERROR_contentChecksum_invalid);
                 nextSrcSizeHint = 0;
-                LZ4F_resetDecompressionContext(dctxPtr);
+                LZ4F_resetDecompressionContext(dctx);
                 doAnotherStage = 0;
                 break;
             }
@@ -1497,80 +1569,77 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
                 srcPtr += 4;
             } else {
                 /* not enough input to read cBlockSize field */
-                dctxPtr->tmpInSize = 4;
-                dctxPtr->tmpInTarget = 8;
-                dctxPtr->dStage = dstage_storeSFrameSize;
+                dctx->tmpInSize = 4;
+                dctx->tmpInTarget = 8;
+                dctx->dStage = dstage_storeSFrameSize;
             }
 
-            if (dctxPtr->dStage == dstage_storeSFrameSize)
+            if (dctx->dStage == dstage_storeSFrameSize)
         case dstage_storeSFrameSize:
             {
-                size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd - srcPtr))
-                    sizeToCopy = srcEnd - srcPtr;
-                memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+                size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
+                                             (size_t)(srcEnd - srcPtr) );
+                memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
-                dctxPtr->tmpInSize += sizeToCopy;
-                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) {
+                dctx->tmpInSize += sizeToCopy;
+                if (dctx->tmpInSize < dctx->tmpInTarget) {
                     /* not enough input to get full sBlockSize; wait for more */
-                    nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
+                    nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
                     doAnotherStage = 0;
                     break;
                 }
-                selectedIn = dctxPtr->header + 4;
+                selectedIn = dctx->header + 4;
             }
 
         /* case dstage_decodeSFrameSize: */   /* no direct access */
             {   size_t const SFrameSize = LZ4F_readLE32(selectedIn);
-                dctxPtr->frameInfo.contentSize = SFrameSize;
-                dctxPtr->tmpInTarget = SFrameSize;
-                dctxPtr->dStage = dstage_skipSkippable;
+                dctx->frameInfo.contentSize = SFrameSize;
+                dctx->tmpInTarget = SFrameSize;
+                dctx->dStage = dstage_skipSkippable;
                 break;
             }
 
         case dstage_skipSkippable:
-            {   size_t skipSize = dctxPtr->tmpInTarget;
-                if (skipSize > (size_t)(srcEnd-srcPtr))
-                    skipSize = srcEnd-srcPtr;
+            {   size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
                 srcPtr += skipSize;
-                dctxPtr->tmpInTarget -= skipSize;
+                dctx->tmpInTarget -= skipSize;
                 doAnotherStage = 0;
-                nextSrcSizeHint = dctxPtr->tmpInTarget;
-                if (nextSrcSizeHint) break;
-                LZ4F_resetDecompressionContext(dctxPtr);
+                nextSrcSizeHint = dctx->tmpInTarget;
+                if (nextSrcSizeHint) break;  /* still more to skip */
+                LZ4F_resetDecompressionContext(dctx);
                 break;
             }
         }
     }
 
     /* preserve history within tmp if necessary */
-    if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked)
-      && (dctxPtr->dict != dctxPtr->tmpOutBuffer)
-      && (dctxPtr->dStage != dstage_getHeader)
+    if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
+      && (dctx->dict != dctx->tmpOutBuffer)
+      && (dctx->dStage != dstage_getFrameHeader)
       && (!decompressOptionsPtr->stableDst)
-      && ((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
+      && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
     {
-        if (dctxPtr->dStage == dstage_flushOut) {
-            size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
-            size_t copySize = 64 KB - dctxPtr->tmpOutSize;
-            const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
-            if (dctxPtr->tmpOutSize > 64 KB) copySize = 0;
+        if (dctx->dStage == dstage_flushOut) {
+            size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+            size_t copySize = 64 KB - dctx->tmpOutSize;
+            const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
+            if (dctx->tmpOutSize > 64 KB) copySize = 0;
             if (copySize > preserveSize) copySize = preserveSize;
 
-            memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+            memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
 
-            dctxPtr->dict = dctxPtr->tmpOutBuffer;
-            dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart;
+            dctx->dict = dctx->tmpOutBuffer;
+            dctx->dictSize = preserveSize + dctx->tmpOutStart;
         } else {
-            size_t newDictSize = dctxPtr->dictSize;
-            const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
+            size_t newDictSize = dctx->dictSize;
+            const BYTE* oldDictEnd = dctx->dict + dctx->dictSize;
             if ((newDictSize) > 64 KB) newDictSize = 64 KB;
 
-            memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+            memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
 
-            dctxPtr->dict = dctxPtr->tmpOutBuffer;
-            dctxPtr->dictSize = newDictSize;
-            dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
+            dctx->dict = dctx->tmpOutBuffer;
+            dctx->dictSize = newDictSize;
+            dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
         }
     }
 
@@ -1584,17 +1653,17 @@ size_t LZ4F_decompress(LZ4F_dctx* dctxPtr,
  *  Dictionary is used "in place", without any preprocessing.
  *  It must remain accessible throughout the entire frame decoding.
  */
-size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
+size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
                        void* dstBuffer, size_t* dstSizePtr,
                        const void* srcBuffer, size_t* srcSizePtr,
                        const void* dict, size_t dictSize,
                        const LZ4F_decompressOptions_t* decompressOptionsPtr)
 {
-    if (dctxPtr->dStage <= dstage_init) {
-        dctxPtr->dict = (const BYTE*)dict;
-        dctxPtr->dictSize = dictSize;
+    if (dctx->dStage <= dstage_init) {
+        dctx->dict = (const BYTE*)dict;
+        dctx->dictSize = dictSize;
     }
-    return LZ4F_decompress(dctxPtr, dstBuffer, dstSizePtr,
+    return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
                            srcBuffer, srcSizePtr,
                            decompressOptionsPtr);
 }
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 844255b..88a6513 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -140,6 +140,11 @@ typedef enum {
 } LZ4F_contentChecksum_t;
 
 typedef enum {
+    LZ4F_noBlockChecksum=0,
+    LZ4F_blockChecksumEnabled
+} LZ4F_blockChecksum_t;
+
+typedef enum {
     LZ4F_frame=0,
     LZ4F_skippableFrame
     LZ4F_OBSOLETE_ENUM(skippableFrame)
@@ -153,23 +158,23 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
 #endif
 
 /*! LZ4F_frameInfo_t :
- * makes it possible to supply detailed frame parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+ *  makes it possible to set or read frame parameters.
+ *  It's not required to set all fields, as long as the structure was initially memset() to zero.
+ *  For all fields, 0 sets it to default value */
 typedef struct {
   LZ4F_blockSizeID_t     blockSizeID;          /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  LZ4F_blockMode_t       blockMode;            /* blockLinked, blockIndependent ; 0 == default */
-  LZ4F_contentChecksum_t contentChecksumFlag;  /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  LZ4F_frameType_t       frameType;            /* LZ4F_frame, skippableFrame ; 0 == default */
+  LZ4F_blockMode_t       blockMode;            /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;  /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default)  */
+  LZ4F_frameType_t       frameType;            /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
   unsigned long long     contentSize;          /* Size of uncompressed content ; 0 == unknown */
   unsigned               dictID;               /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
-  unsigned               reserved[1];          /* must be zero for forward compatibility */
+  LZ4F_blockChecksum_t   blockChecksumFlag;    /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default)  */
 } LZ4F_frameInfo_t;
 
 /*! LZ4F_preferences_t :
- * makes it possible to supply detailed compression parameters to the stream interface.
- * It's not required to set all fields, as long as the structure was initially memset() to zero.
- * All reserved fields must be set to zero. */
+ *  makes it possible to supply detailed compression parameters to the stream interface.
+ *  It's not required to set all fields, as long as the structure was initially memset() to zero.
+ *  All reserved fields must be set to zero. */
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
   int      compressionLevel;       /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */
diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index b585b72..1899f8e 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -59,7 +59,7 @@ extern "C" {
         ITEM(ERROR_contentChecksumFlag_invalid) \
         ITEM(ERROR_compressionLevel_invalid) \
         ITEM(ERROR_headerVersion_wrong) \
-        ITEM(ERROR_blockChecksum_unsupported) \
+        ITEM(ERROR_blockChecksum_invalid) \
         ITEM(ERROR_reservedFlag_set) \
         ITEM(ERROR_allocation_failed) \
         ITEM(ERROR_srcSize_tooLarge) \
diff --git a/tests/frametest.c b/tests/frametest.c
index d0665c5..88d0afd 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -402,7 +402,7 @@ int basicTests(U32 seed, double compressibility)
     CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
-    DISPLAYLEVEL(3, "without checksum : ");
+    DISPLAYLEVEL(3, "without frame checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
     CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@@ -416,7 +416,7 @@ int basicTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
     }
 
-    DISPLAYLEVEL(3, "without checksum : ");
+    DISPLAYLEVEL(3, "without frame checksum : ");
     prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
     {   size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
         DISPLAYLEVEL(4, "dstCapacity = %u  ; ", (U32)dstCapacity)
@@ -424,6 +424,29 @@ int basicTests(U32 seed, double compressibility)
         DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
     }
 
+    DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
+    memset(&prefs, 0, sizeof(prefs));
+    prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
+    CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
+    DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
+
+    DISPLAYLEVEL(3, "Decompress with block checksum : ");
+    {   size_t iSize = cSize;
+        size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
+        LZ4F_decompressionContext_t dctx;
+        CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
+        CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
+        if (decodedSize != testSize) goto _output_error;
+        if (iSize != cSize) goto _output_error;
+        {   U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
+            U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
+            if (crcDest != crcSrc) goto _output_error;
+        }
+        DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
+
+        CHECK( LZ4F_freeDecompressionContext(dctx) );
+    }
+
     /* frame content size tests */
     {   size_t cErr;
         BYTE* const ostart = (BYTE*)compressedBuffer;
@@ -771,6 +794,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         memset(&prefs, 0, sizeof(prefs));
         prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
         prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
+        prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
         prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
         prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
         prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
-- 
cgit v0.12


From 930a6921103c3ae4e62c99f86fd5329ef957900f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 11 Aug 2017 17:58:46 -0700
Subject: cli : restored command -BX to enable block checksum (#322)

---
 programs/lz4cli.c |  2 +-
 programs/lz4io.c  | 11 ++++++-----
 tests/Makefile    |  1 +
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 55f3133..dede834 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -144,7 +144,7 @@ static int usage_advanced(const char* exeName)
     DISPLAY( " -l     : compress using Legacy format (Linux kernel compression)\n");
     DISPLAY( " -B#    : Block size [4-7] (default : 7) \n");
     DISPLAY( " -BD    : Block dependency (improve compression ratio) \n");
-    /* DISPLAY( " -BX    : enable block checksum (default:disabled)\n");   *//* Option currently inactive */
+    DISPLAY( " -BX    : enable block checksum (default:disabled) \n");
     DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
     DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 1e6b437..06741b4 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -173,17 +173,17 @@ int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
     return g_blockIndependence;
 }
 
-/* Default setting : no checksum */
-int LZ4IO_setBlockChecksumMode(int xxhash)
+/* Default setting : no block checksum */
+int LZ4IO_setBlockChecksumMode(int enable)
 {
-    g_blockChecksum = (xxhash != 0);
+    g_blockChecksum = (enable != 0);
     return g_blockChecksum;
 }
 
 /* Default setting : checksum enabled */
-int LZ4IO_setStreamChecksumMode(int xxhash)
+int LZ4IO_setStreamChecksumMode(int enable)
 {
-    g_streamChecksum = (xxhash != 0);
+    g_streamChecksum = (enable != 0);
     return g_streamChecksum;
 }
 
@@ -455,6 +455,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     prefs.compressionLevel = compressionLevel;
     prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
     prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
+    prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
     prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
     if (g_contentSizeFlag) {
       U64 const fileSize = UTIL_getFileSize(srcFileName);
diff --git a/tests/Makefile b/tests/Makefile
index 5d2532f..f00778f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -263,6 +263,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	$(LZ4) -f tmp
 	cat tmp >> tmp.lz4
 	$(LZ4) -f tmp.lz4         # uncompress valid frame followed by invalid data
+	$(LZ4) -BX tmp -c -q | $(LZ4) -tv  # test block checksum
 	@$(RM) tmp*
 
 test-lz4-hugefile: lz4 datagen
-- 
cgit v0.12


From aea7d521aed2506270a14d8e901ea657d6a1faf4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 14 Aug 2017 15:13:23 -0700
Subject: better respect GNU standard Makefile conventions

supports lowercase directory variables
add an "Installation" section in README.md
added an INSTALL file
---
 INSTALL                  | 13 +++++++++++++
 Makefile                 | 22 +++++++++-------------
 README.md                | 17 ++++++++++++++++-
 doc/lz4frame_manual.html | 32 +++++++++++++++++---------------
 lib/Makefile             | 40 +++++++++++++++++++++++-----------------
 programs/Makefile        | 41 ++++++++++++++++++++++++-----------------
 6 files changed, 102 insertions(+), 63 deletions(-)
 create mode 100644 INSTALL

diff --git a/INSTALL b/INSTALL
new file mode 100644
index 0000000..8f61221
--- /dev/null
+++ b/INSTALL
@@ -0,0 +1,13 @@
+Installation
+=============
+
+`make`
+`make install`     # this command may require root access
+
+LZ4's `Makefile` supports standard [Makefile conventions],
+including [staged installs], [directory redirection], or [command redefinition].
+
+[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
+[directory redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
+[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
diff --git a/Makefile b/Makefile
index b31ebd0..972d0b2 100644
--- a/Makefile
+++ b/Makefile
@@ -32,12 +32,6 @@
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
-DESTDIR ?=
-PREFIX  ?= /usr/local
-VOID    := /dev/null
-
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
 LZ4DIR  = lib
 PRGDIR  = programs
 TESTDIR = tests
@@ -46,14 +40,16 @@ EXDIR   = examples
 
 # Define nul output
 ifneq (,$(filter Windows%,$(OS)))
-EXT = .exe
+EXT  = .exe
+VOID = nul
 else
-EXT =
+EXT  =
+VOID = /dev/null
 endif
 
 
 .PHONY: default
-default: lib lz4-release
+default: lib-release lz4-release
 
 .PHONY: all
 all: allmost manuals
@@ -61,9 +57,9 @@ all: allmost manuals
 .PHONY: allmost
 allmost: lib lz4 examples
 
-.PHONY: lib
-lib:
-	@$(MAKE) -C $(LZ4DIR)
+.PHONY: lib lib-release
+lib lib-release:
+	@$(MAKE) -C $(LZ4DIR) $@
 
 .PHONY: lz4 lz4-release
 lz4 lz4-release: lib
@@ -100,7 +96,7 @@ install uninstall:
 	@$(MAKE) -C $(PRGDIR) $@
 
 travis-install:
-	$(MAKE) -j1 install PREFIX=~/install_test_dir
+	$(MAKE) -j1 install DESTDIR=~/install_test_dir
 
 cmake:
 	@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
diff --git a/README.md b/README.md
index 950e4c4..0e469c8 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,22 @@ in single-thread mode.
 [zlib]: http://www.zlib.net/
 [Zstandard]: http://www.zstd.net/
 
-LZ4 is also compatible and well optimized for x32 mode, for which it provides +10% speed performance.
+LZ4 is also compatible and well optimized for x32 mode, for which it provides an additional +10% speed performance.
+
+
+Installation
+-------------------------
+
+`make`
+`make install`     # this command may require root access
+
+LZ4's `Makefile` supports standard [Makefile conventions],
+including [staged installs], [directory redirection], or [command redefinition].
+
+[Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+[staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
+[directory redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
+[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
 
 
 Documentation
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 87750a1..b82dfe5 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -186,15 +186,15 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
Decompression functions

 
 
typedef struct {
-  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned stableDst;    /* pledge that at least 64KB+64Bytes of previously decompressed data remain unmodifed where it was decoded. This optimization skips storage operations in tmp buffers */
   unsigned reserved[3];
 } LZ4F_decompressOptions_t;
 


 
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version);
 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
-
 Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations.
- The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
- The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+
 Create an LZ4F_dctx object, to track all decompression operations.
+ The version provided MUST be LZ4F_VERSION.
+ The function provides a pointer to an allocated and initialized LZ4F_dctx object.
  The result is an errorCode, which can be tested using LZ4F_isError().
  dctx memory can be released using LZ4F_freeDecompressionContext();
  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
@@ -208,20 +208,22 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                      LZ4F_frameInfo_t* frameInfoPtr,
                                      const void* srcBuffer, size_t* srcSizePtr);
 
 This function extracts frame parameters (such as max blockSize, frame checksum, etc.).
- Its usage is optional. Extracted information can be useful for allocation purposes, typically.
+ Its usage is optional.
+ Extracted information can typically be useful for allocation purposes.
  This function works in 2 situations :
-   - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
+   - At the beginning of a new frame, in which case
+     it will decode information from `srcBuffer`, starting the decoding process.
      Input size must be large enough to successfully decode the entire frame header.
      Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes.
      It's allowed to provide more input data than this minimum.
    - After decoding has been started.
      In which case, no input is read, frame parameters are extracted from dctx.
-     If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
+   - If decoding has barely started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail.
  The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
  Decompression must resume from (srcBuffer + *srcSizePtr).
  @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
            or an error code which can be tested using LZ4F_isError()
- note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
+ note 1 : in case of error, dctx is not modified. Decoding operation can resume safely.
  note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
  
 


@@ -230,18 +232,18 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                    void* dstBuffer, size_t* dstSizePtr,
                                    const void* srcBuffer, size_t* srcSizePtr,
                                    const LZ4F_decompressOptions_t* dOptPtr);
-
 Call this function repetitively to regenerate data compressed within `srcBuffer`.
+
 Call this function repetitively to regenerate compressed data from `srcBuffer`.
  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
 
- The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
 
- The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
- Number of bytes read can be < number of bytes provided, meaning there is some more data to decode.
+ The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
+ Number of bytes consumed can be < number of bytes provided.
  It typically happens when dstBuffer is not large enough to contain all decoded data.
- Remaining data will have to be presented again in a subsequent invocation.
+ Unconsumed source data must be presented again in subsequent invocations.
 
  `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- `dstBuffer` can be changed at will between each consecutive function invocation.
+ `dstBuffer` itself can be changed at will between each consecutive function invocation.
 
  @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
@@ -259,7 +261,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
  In case of an error, the context is left in "undefined" state.
   In which case, it's necessary to reset it, before re-using it.
   This method can also be used to abruptly stop an unfinished decompression,
-  and start a new with the same context. 
+  and start a new one using the same context. 
 


 
 
diff --git a/lib/Makefile b/lib/Makefile
index de5c4ff..9abb699 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -60,7 +60,7 @@ ifeq ($(shell uname), Darwin)
 	SHARED_EXT = dylib
 	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
 	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
-	SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+	SONAME_FLAGS = -install_name $(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
 	SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR)
 	SHARED_EXT = so
@@ -115,16 +115,16 @@ clean:
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-PREFIX     ?= /usr/local
-DESTDIR    ?=
-LIBDIR     ?= $(PREFIX)/lib
-INCLUDEDIR ?= $(PREFIX)/include
+DESTDIR     ?=
+# directory variables : GNU convention prefers lowercase
+# support both lower and uppercase (BSD), use uppercase in script
+prefix      ?= /usr/local
+PREFIX      ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+libdir      ?= $(exec_prefix)/lib
+LIBDIR      ?= $(libdir)
+includedir  ?= $(PREFIX)/include
+INCLUDEDIR  ?= $(includedir)
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
 PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
@@ -132,8 +132,14 @@ else
 PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
-INSTALL_LIB  ?= $(INSTALL) -m 755
-INSTALL_DATA ?= $(INSTALL) -m 644
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL)
+INSTALL_DATA    ?= $(INSTALL) -m 644
 
 liblz4.pc: liblz4.pc.in Makefile
 	@echo creating pkgconfig
@@ -148,17 +154,17 @@ install: lib liblz4.pc
 	@$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/
 	@echo Installing libraries
 ifeq ($(BUILD_STATIC),yes)
-	@$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
+	@$(INSTALL_DATA) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
 	@$(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(INCLUDEDIR)/lz4frame_static.h
 endif
-	@$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
+	@$(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
 	@ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-	@echo Installing includes
+	@echo Installing headers in $(INCLUDEDIR)
 	@$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
 	@$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
-	@echo lz4 static and shared libraries installed
+	@echo lz4 libraries installed
 
 uninstall:
 	@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc
diff --git a/programs/Makefile b/programs/Makefile
index c484731..224b013 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,8 @@
 # ##########################################################################
 
 # Version numbers
-LIBVER_SRC := ../lib/lz4.h
+LZ4DIR   := ../lib
+LIBVER_SRC := $(LZ4DIR)/lz4.h
 LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
 LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
 LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
@@ -38,32 +39,31 @@ LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCR
 LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
 LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
-LIBVER  := $(shell echo $(LIBVER_SCRIPT))
-
-LZ4DIR  := ../lib
+LIBVER   := $(shell echo $(LIBVER_SCRIPT))
 
 SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)
 OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
-VOID := /dev/null
 
-CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
-CFLAGS  ?= -O3
+CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
+CFLAGS   ?= -O3
 DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
+CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
+FLAGS     = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 LZ4_VERSION=$(LIBVER)
-MD2ROFF  =ronn
-MD2ROFF_FLAGS  = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
+MD2ROFF   = ronn
+MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)"
 
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT  :=.exe
+VOID := nul
 else
 EXT  :=
+VOID := /dev/null
 endif
 
 
@@ -121,18 +121,25 @@ else
 INSTALL ?= install
 endif
 
-PREFIX  ?= /usr/local
-DESTDIR ?=
-BINDIR  ?= $(PREFIX)/bin
+DESTDIR     ?=
+# directory variables : GNU convention prefers lowercase
+# support both lower and uppercase (BSD), use uppercase in script
+prefix      ?= /usr/local
+PREFIX      ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+bindir      ?= $(exec_prefix)/bin
+BINDIR      ?= $(bindir)
+datarootdir ?= $(PREFIX)/share
+mandir      ?= $(datarootdir)/man
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
 MANDIR  ?= $(PREFIX)/man/man1
 else
-MANDIR  ?= $(PREFIX)/share/man/man1
+MANDIR  ?= $(mandir)
 endif
 
 INSTALL_PROGRAM ?= $(INSTALL) -m 755
-INSTALL_MAN     ?= $(INSTALL) -m 644
+INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
 install: lz4$(EXT) lz4c$(EXT)
@@ -143,7 +150,7 @@ install: lz4$(EXT) lz4c$(EXT)
 	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
 	@$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c
 	@echo Installing man pages
-	@$(INSTALL_MAN) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
+	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
-- 
cgit v0.12


From 731cff12080f1309af4abde122fe09ae0076f424 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 14 Aug 2017 15:18:47 -0700
Subject: fix minor markdown display issues

---
 INSTALL   | 10 ++++++----
 README.md | 10 ++++++----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/INSTALL b/INSTALL
index 8f61221..f5ac0a0 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,13 +1,15 @@
 Installation
 =============
 
-`make`
-`make install`     # this command may require root access
+```
+make
+make install     # this command may require root access
+```
 
 LZ4's `Makefile` supports standard [Makefile conventions],
-including [staged installs], [directory redirection], or [command redefinition].
+including [staged installs], [redirection], or [command redefinition].
 
 [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
-[directory redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
+[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
 [command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
diff --git a/README.md b/README.md
index 0e469c8..ec99630 100644
--- a/README.md
+++ b/README.md
@@ -75,15 +75,17 @@ LZ4 is also compatible and well optimized for x32 mode, for which it provides an
 Installation
 -------------------------
 
-`make`
-`make install`     # this command may require root access
+```
+make
+make install     # this command may require root access
+```
 
 LZ4's `Makefile` supports standard [Makefile conventions],
-including [staged installs], [directory redirection], or [command redefinition].
+including [staged installs], [redirection], or [command redefinition].
 
 [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
-[directory redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
+[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
 [command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
 
 
-- 
cgit v0.12


From 01cdbfb5feda4adee65cc3da04b426c0c22368a3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 14 Aug 2017 16:56:05 -0700
Subject: lz4c legacy commands are now enabled at runtime based on link/binary
 name "lz4c"

instead of selected at compilation time depending on a macro.
This design makes it possible to have a single binary which supports both modes.

An advantageous side effect is that when doing `make; make install`
no additional binary is created during `make install`
(it used to create `lz4c`, because `make` would only build `lz4`)
---
 Makefile          |  5 ++++-
 programs/Makefile |  6 +++--
 programs/lz4cli.c | 66 ++++++++++++++++++++++++++-----------------------------
 3 files changed, 39 insertions(+), 38 deletions(-)

diff --git a/Makefile b/Makefile
index 972d0b2..da485a1 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,9 @@ lib lib-release:
 	@$(MAKE) -C $(LZ4DIR) $@
 
 .PHONY: lz4 lz4-release
-lz4 lz4-release: lib
+lz4 : lib
+lz4-release : lib-release
+lz4 lz4-release :
 	@$(MAKE) -C $(PRGDIR) $@
 	@cp $(PRGDIR)/lz4$(EXT) .
 
@@ -91,6 +93,7 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
 
+.PHONY: install uninstall
 install uninstall:
 	@$(MAKE) -C $(LZ4DIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
diff --git a/programs/Makefile b/programs/Makefile
index 224b013..c7ef6d1 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -82,10 +82,12 @@ lz4-release: DEBUGFLAGS=
 lz4-release: lz4
 
 lz4c32: CFLAGS += -m32
-lz4c: CPPFLAGS += -DENABLE_LZ4C_LEGACY_OPTIONS
-lz4c lz4c32 : $(SRCFILES)
+lz4c32 : $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
+lz4c: lz4
+	ln -s lz4 lz4c
+
 lz4.1: lz4.1.md
 	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 55f3133..0cabc27 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -30,14 +30,6 @@
   The license of this compression CLI program is GPLv2.
 */
 
-/**************************************
-*  Tuning parameters
-***************************************/
-/* ENABLE_LZ4C_LEGACY_OPTIONS :
-   Control the availability of -c0, -c1 and -hc legacy arguments
-   Default : Legacy options are disabled */
-/* #define ENABLE_LZ4C_LEGACY_OPTIONS */
-
 
 /****************************
 *  Includes
@@ -62,6 +54,8 @@
 #define LZ4_EXTENSION ".lz4"
 #define LZ4CAT "lz4cat"
 #define UNLZ4 "unlz4"
+#define LZ4_LEGACY "lz4c"
+static int g_lz4c_legacy_commands = 0;
 
 #define KB *(1U<<10)
 #define MB *(1U<<20)
@@ -154,13 +148,13 @@ static int usage_advanced(const char* exeName)
     DISPLAY( " -i#    : minimum evaluation time in seconds (default : 3s) \n");
     DISPLAY( " -B#    : cut file into independent blocks of size # bytes [32+] \n");
     DISPLAY( "                     or predefined block size [4-7] (default: 7) \n");
-#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
-    DISPLAY( "Legacy arguments : \n");
-    DISPLAY( " -c0    : fast compression \n");
-    DISPLAY( " -c1    : high compression \n");
-    DISPLAY( " -hc    : high compression \n");
-    DISPLAY( " -y     : overwrite output without prompting \n");
-#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
+    if (g_lz4c_legacy_commands) {
+        DISPLAY( "Legacy arguments : \n");
+        DISPLAY( " -c0    : fast compression \n");
+        DISPLAY( " -c1    : high compression \n");
+        DISPLAY( " -hc    : high compression \n");
+        DISPLAY( " -y     : overwrite output without prompting \n");
+    }
     EXTENDED_HELP;
     return 0;
 }
@@ -212,17 +206,17 @@ static int usage_longhelp(const char* exeName)
     DISPLAY( "-------------------------------------\n");
     DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
     DISPLAY( "          generator | %s | consumer \n", exeName);
-#if defined(ENABLE_LZ4C_LEGACY_OPTIONS)
-    DISPLAY( "\n");
-    DISPLAY( "***** Warning  *****\n");
-    DISPLAY( "Legacy arguments take precedence. Therefore : \n");
-    DISPLAY( "---------------------------------\n");
-    DISPLAY( "          %s -hc filename\n", exeName);
-    DISPLAY( "means 'compress filename in high compression mode'\n");
-    DISPLAY( "It is not equivalent to :\n");
-    DISPLAY( "          %s -h -c filename\n", exeName);
-    DISPLAY( "which would display help text and exit\n");
-#endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
+    if (g_lz4c_legacy_commands) {
+        DISPLAY( "\n");
+        DISPLAY( "***** Warning  *****\n");
+        DISPLAY( "Legacy arguments take precedence. Therefore : \n");
+        DISPLAY( "---------------------------------\n");
+        DISPLAY( "          %s -hc filename\n", exeName);
+        DISPLAY( "means 'compress filename in high compression mode'\n");
+        DISPLAY( "It is not equivalent to :\n");
+        DISPLAY( "          %s -h -c filename\n", exeName);
+        DISPLAY( "which would display help text and exit\n");
+    }
     return 0;
 }
 
@@ -314,7 +308,7 @@ int main(int argc, const char** argv)
     inFileNames[0] = stdinmark;
     LZ4IO_setOverwrite(0);
 
-    /* lz4cat predefined behavior */
+    /* predefined behaviors, based on binary/link name */
     if (exeNameMatch(exeName, LZ4CAT)) {
         mode = om_decompress;
         LZ4IO_setOverwrite(1);
@@ -325,6 +319,7 @@ int main(int argc, const char** argv)
         multiple_inputs=1;
     }
     if (exeNameMatch(exeName, UNLZ4)) { mode = om_decompress; }
+    if (exeNameMatch(exeName, LZ4_LEGACY)) { g_lz4c_legacy_commands=1; }
 
     /* command switches */
     for(i=1; i='0') && (*argument<='9')) {
                     cLevel = readU32FromChar(&argument);
@@ -530,7 +525,8 @@ int main(int argc, const char** argv)
 #ifdef _FILE_OFFSET_BITS
     DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS);
 #endif
-    if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10));
+    if ((mode == om_compress) || (mode == om_bench))
+        DISPLAYLEVEL(4, "Blocks size : %u KB\n", (U32)(blockSize>>10));
 
     if (multiple_inputs) {
         input_filename = inFileNames[0];
-- 
cgit v0.12


From edb434365942404ec0f86db152ae268d3ce32dd1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 15 Aug 2017 16:58:12 -0700
Subject: updated lz4frame manual

---
 doc/lz4frame_manual.html | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 86aaf3d..7529f6e 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -84,7 +84,7 @@
 } LZ4F_frameInfo_t;
 
  makes it possible to set or read frame parameters.
   It's not required to set all fields, as long as the structure was initially memset() to zero.
-  For all fields, 0 sets it to default value
+  For all fields, 0 sets it to default value 
 


 
 
typedef struct {
@@ -95,7 +95,7 @@
 } LZ4F_preferences_t;
 
  makes it possible to supply detailed compression parameters to the stream interface.
   It's not required to set all fields, as long as the structure was initially memset() to zero.
-  All reserved fields must be set to zero.
+  All reserved fields must be set to zero. 
 


 
 
Simple compression function

@@ -103,7 +103,7 @@
 
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
   Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
-
+ 
 


 
 
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
@@ -114,7 +114,7 @@
   The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
  @return : number of bytes written into dstBuffer.
            or an error code if it fails (can be tested using LZ4F_isError())
-
+ 
 


 
 
Advanced compression functions

@@ -134,7 +134,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  The function will provide a pointer to a fully allocated LZ4F_cctx object.
  If @return != zero, there was an error during context creation.
  Object can release its memory using LZ4F_freeCompressionContext();
-
+ 
 


 
 
Compression

@@ -147,7 +147,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
  @return : number of bytes written into dstBuffer for the header
            or an error code (which can be tested using LZ4F_isError())
-
+ 
 


 
 
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
@@ -155,7 +155,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
-
+ 
 


 
 
size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
@@ -167,7 +167,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
-
+ 
 


 
 
size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
@@ -177,7 +177,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
  @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx)
            or an error code if it fails (which can be tested using LZ4F_isError())
-
+ 
 


 
 
size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
@@ -188,7 +188,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
            or an error code if it fails (which can be tested using LZ4F_isError())
   A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.
-
+ 
 


 
 
Decompression functions

@@ -207,7 +207,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
  dctx memory can be released using LZ4F_freeDecompressionContext();
  The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
  That is, it should be == 0 if decompression has been completed fully and correctly.
-
+ 
 


 
 
Streaming decompression functions

@@ -234,7 +234,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
            or an error code which can be tested using LZ4F_isError().
   note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely.
   note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
-
+ 
 


 
 
size_t LZ4F_decompress(LZ4F_dctx* dctx,
@@ -263,14 +263,14 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 
   After a frame is fully decoded, dctx can be used again to decompress another frame.
   After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
-
+ 
 


 
 
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
 
  In case of an error, the context is left in "undefined" state.
   In which case, it's necessary to reset it, before re-using it.
   This method can also be used to abruptly stop an unfinished decompression,
-  and start a new one using the same context.
+  and start a new one using the same context. 
 


 
 
-- 
cgit v0.12


From c10863b98e1503af90616ae99725ecd120265dfb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Aug 2017 10:55:30 -0700
Subject: fix : asan error in dctx, due to increased maximum frame header size,
 reported by Craig Young

---
 NEWS           | 1 +
 lib/lz4frame.c | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index 63d53a6..e63a82f 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,7 @@ API : added LZ4F_resetDecompressionContext()
 API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
 API : lz4frame : can control block checksum and dictionary ID
 API : fix : expose obsolete decoding functions, reported by Chen Yufei
+API : experimental : lz4frame_static : new dictionary compression API
 build : fix : static lib installation, by Ido Rosen
 build : dragonFlyBSD, OpenBSD, NetBSD supported
 build : LZ4_MEMORY_USAGE can be modified at compile time, through external define
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index e613901..3408708 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -70,7 +70,7 @@ You can contact the author at :
 /*-************************************
 *  Debug
 **************************************/
-#define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4F_STATIC_ASSERT(c)    { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
 
 /*-************************************
@@ -212,7 +212,7 @@ LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
 static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
 {
     /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
-    LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
+    LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
     return (LZ4F_errorCode_t)-(ptrdiff_t)code;
 }
 
@@ -912,7 +912,7 @@ struct LZ4F_dctx_s {
     size_t tmpOutStart;
     XXH32_state_t xxh;
     XXH32_state_t blockChecksum;
-    BYTE   header[16];
+    BYTE   header[LZ4F_HEADER_SIZE_MAX];
 };  /* typedef'd to LZ4F_dctx in lz4frame.h */
 
 
-- 
cgit v0.12


From 5637d8856f2e4ebd04b06e814e65c543621d8813 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Aug 2017 15:26:03 -0700
Subject: make test is compatible with parallel execution (-j#)

each test section runs with its own set of files
---
 README.md      |   1 +
 tests/Makefile | 253 ++++++++++++++++++++++++++++-----------------------------
 2 files changed, 126 insertions(+), 128 deletions(-)

diff --git a/README.md b/README.md
index ec99630..d7e8fb6 100644
--- a/README.md
+++ b/README.md
@@ -82,6 +82,7 @@ make install     # this command may require root access
 
 LZ4's `Makefile` supports standard [Makefile conventions],
 including [staged installs], [redirection], or [command redefinition].
+It is compatible with parallel builds (`'-j#'`).
 
 [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
diff --git a/tests/Makefile b/tests/Makefile
index f00778f..5b30684 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -30,19 +30,14 @@
 # datagen : generates synthetic data samples for tests & benchmarks
 # ##########################################################################
 
-DESTDIR ?=
-PREFIX  ?= /usr/local
-BINDIR  := $(PREFIX)/bin
-MANDIR  := $(PREFIX)/share/man/man1
 LZ4DIR  := ../lib
 PRGDIR  := ../programs
-VOID    := /dev/null
 TESTDIR := versionsTest
 PYTHON  ?= python3
 
 CFLAGS  ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
-CFLAGS  += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-           -Wdeclaration-after-statement -Wstrict-prototypes \
+CFLAGS  += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
+           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(MOREFLAGS)
 CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
@@ -51,9 +46,11 @@ FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
-EXT =.exe
+EXT  =.exe
+VOID = nul
 else
-EXT =
+EXT  =
+VOID = /dev/null
 endif
 LZ4     := $(PRGDIR)/lz4$(EXT)
 
@@ -72,13 +69,13 @@ all32: CFLAGS+=-m32
 all32: all
 
 lz4:
-	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
+	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
 
 lz4c:
-	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="$(CFLAGS)"
+	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
 
 lz4c32:   # create a 32-bits version for 32/64 interop tests
-	$(MAKE) -C $(PRGDIR) clean $@ CFLAGS="-m32 $(CFLAGS)"
+	$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
 	cp $(LZ4) $(LZ4)c32
 
 fullbench  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
@@ -144,75 +141,75 @@ test32: test
 
 test-lz4-sparse: lz4 datagen
 	@echo "\n ---- test sparse file support ----"
-	./datagen -g5M  -P100 > tmpSrc
-	$(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4
-	$(DIFF) -s tmpSrc tmpB4
-	$(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5
-	$(DIFF) -s tmpSrc tmpB5
-	$(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6
-	$(DIFF) -s tmpSrc tmpB6
-	$(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7
-	$(DIFF) -s tmpSrc tmpB7
-	$(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse
-	$(DIFF) -s tmpSrc tmpNoSparse
-	ls -ls tmp*
-	./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd   # Odd size file (to generate non-full last block)
-	./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmpOdd
-	ls -ls tmpOdd
-	@$(RM) tmp*
+	./datagen -g5M  -P100 > tmplsdg5M
+	$(LZ4) -B4D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB4
+	$(DIFF) -s tmplsdg5M tmplscB4
+	$(LZ4) -B5D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB5
+	$(DIFF) -s tmplsdg5M tmplscB5
+	$(LZ4) -B6D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB6
+	$(DIFF) -s tmplsdg5M tmplscB6
+	$(LZ4) -B7D tmplsdg5M | $(LZ4) -dv --sparse > tmplscB7
+	$(DIFF) -s tmplsdg5M tmplscB7
+	$(LZ4) tmplsdg5M | $(LZ4) -dv --no-sparse > tmplsnosparse
+	$(DIFF) -s tmplsdg5M tmplsnosparse
+	ls -ls tmpls*
+	./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd   # Odd size file (to generate non-full last block)
+	./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
+	ls -ls tmplsodd
+	@$(RM) tmpls*
 	@echo "\n Compatibility with Console :"
 	echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
 	echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | cat
 	echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
 	@echo "\n Compatibility with Append :"
-	./datagen -P100 -g1M > tmp1M
-	cat tmp1M tmp1M > tmp2M
-	$(LZ4) -B5 -v tmp1M tmpC
-	$(LZ4) -d -v tmpC tmpR
-	$(LZ4) -d -v tmpC >> tmpR
+	./datagen -P100 -g1M > tmplsdg1M
+	cat tmplsdg1M tmplsdg1M > tmpls2M
+	$(LZ4) -B5 -v tmplsdg1M tmplsc
+	$(LZ4) -d -v tmplsc tmplsr
+	$(LZ4) -d -v tmplsc >> tmplsr
 	ls -ls tmp*
-	$(DIFF) tmp2M tmpR
-	@$(RM) tmp*
+	$(DIFF) tmpls2M tmplsr
+	@$(RM) tmpls*
 
 test-lz4-contentSize: lz4 datagen
 	@echo "\n ---- test original size support ----"
-	./datagen -g15M > tmp
-	$(LZ4) -v tmp | $(LZ4) -t
-	$(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2
-	$(DIFF) -s tmp tmp2
+	./datagen -g15M > tmplc1
+	$(LZ4) -v tmplc1 | $(LZ4) -t
+	$(LZ4) -v --content-size tmplc1 | $(LZ4) -d > tmplc2
+	$(DIFF) -s tmplc1 tmplc2
 	# test large size [2-4] GB
-	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp
-	@ls -ls tmp
-	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2
-	@ls -ls tmp2
-	$(DIFF) -s tmp tmp2
-	@$(RM) tmp*
+	@./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmplc1
+	@ls -ls tmplc1
+	@./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmplc2
+	@ls -ls tmplc2
+	$(DIFF) -s tmplc1 tmplc2
+	@$(RM) tmplc*
 
 test-lz4-frame-concatenation: lz4 datagen
 	@echo "\n ---- test frame concatenation ----"
-	@echo -n > empty.test
-	@echo hi > nonempty.test
-	cat nonempty.test empty.test nonempty.test > orig.test
-	@$(LZ4) -zq empty.test > empty.lz4.test
-	@$(LZ4) -zq nonempty.test > nonempty.lz4.test
-	cat nonempty.lz4.test empty.lz4.test nonempty.lz4.test > concat.lz4.test
-	$(LZ4) -d concat.lz4.test > result.test
-	sdiff orig.test result.test
-	@$(RM) *.test
+	@echo -n > tmp-lfc-empty
+	@echo hi > tmp-lfc-nonempty
+	cat tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
+	@$(LZ4) -zq tmp-lfc-empty > tmp-lfc-empty.lz4
+	@$(LZ4) -zq tmp-lfc-nonempty > tmp-lfc-nonempty.lz4
+	cat tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
+	$(LZ4) -d tmp-lfc-concat.lz4 > tmp-lfc-result
+	sdiff tmp-lfc-src tmp-lfc-result
+	@$(RM) tmp-lfc-*
 	@echo frame concatenation test completed
 
 test-lz4-multiple: lz4 datagen
 	@echo "\n ---- test multiple files ----"
-	@./datagen -s1        > tmp1 2> $(VOID)
-	@./datagen -s2 -g100K > tmp2 2> $(VOID)
-	@./datagen -s3 -g1M   > tmp3 2> $(VOID)
-	$(LZ4) -f -m tmp*
-	ls -ls tmp*
-	@$(RM) tmp1 tmp2 tmp3
-	$(LZ4) -df -m *.lz4
-	ls -ls tmp*
-	$(LZ4) -f -m tmp1 notHere tmp2; echo $$?
-	@$(RM) tmp*
+	@./datagen -s1        > tmp-tlm1 2> $(VOID)
+	@./datagen -s2 -g100K > tmp-tlm2 2> $(VOID)
+	@./datagen -s3 -g1M   > tmp-tlm3 2> $(VOID)
+	$(LZ4) -f -m tmp-tlm*
+	ls -ls tmp-tlm*
+	@$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
+	$(LZ4) -df -m tmp-tlm*.lz4
+	ls -ls tmp-tlm*
+	$(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$?
+	@$(RM) tmp-tlm*
 
 unlz4:
 	@$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
@@ -222,74 +219,73 @@ lz4cat:
 
 test-lz4-basic: lz4 datagen unlz4 lz4cat
 	@echo "\n ---- test lz4 basic compression/decompression ----"
-	./datagen -g0     | $(LZ4) -v     | $(LZ4) -t
-	./datagen -g16KB  | $(LZ4) -9     | $(LZ4) -t
-	./datagen -g20KB > tmpSrc
-	$(LZ4) < tmpSrc   | $(LZ4) -d > tmpRes
-	$(DIFF) -q tmpSrc tmpRes
-	$(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes
-	$(DIFF) -q tmpSrc tmpRes
-	./datagen         | $(LZ4)        | $(LZ4) -t
-	./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
-	./datagen -g17M   | $(LZ4) -9v    | $(LZ4) -qt
-	./datagen -g33M   | $(LZ4) --no-frame-crc | $(LZ4) -t
-	./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
-	@echo "hello world" > tmp
-	$(LZ4) --rm -f tmp tmp.lz4
-	test ! -f tmp                      # must fail (--rm)
-	test   -f tmp.lz4
-	$(PRGDIR)/lz4cat tmp.lz4           # must display hello world
-	test   -f tmp.lz4
-	$(PRGDIR)/unlz4 --rm tmp.lz4 tmp
-	test   -f tmp
-	test ! -f tmp.lz4                  # must fail (--rm)
-	test ! -f tmp.lz4.lz4              # must fail (unlz4)
-	$(PRGDIR)/lz4cat tmp               # pass-through mode
-	test   -f tmp
-	test ! -f tmp.lz4                  # must fail (lz4cat)
-	$(LZ4) tmp tmp.lz4                 # creates tmp.lz4
-	$(PRGDIR)/lz4cat < tmp.lz4 > tmp3  # checks lz4cat works with stdin (#285)
-	$(DIFF) -q tmp tmp3
-	$(PRGDIR)/lz4cat < tmp > tmp2      # checks lz4cat works with stdin (#285)
-	$(DIFF) -q tmp tmp2
-	cp tmp ./-d
-	$(LZ4) --rm -- -d -d.lz4           # compresses ./d into ./-d.lz4
+	./datagen -g0       | $(LZ4) -v     | $(LZ4) -t
+	./datagen -g16KB    | $(LZ4) -9     | $(LZ4) -t
+	./datagen -g20KB > tmp-tlb-dg20k
+	$(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
+	$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
+	$(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
+	$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
+	./datagen           | $(LZ4)        | $(LZ4) -t
+	./datagen -g6M -P99 | $(LZ4) -9BD   | $(LZ4) -t
+	./datagen -g17M     | $(LZ4) -9v    | $(LZ4) -qt
+	./datagen -g33M     | $(LZ4) --no-frame-crc | $(LZ4) -t
+	./datagen -g256MB   | $(LZ4) -vqB4D | $(LZ4) -t
+	@echo "hello world" > tmp-tlb-hw
+	$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
+	test ! -f tmp-tlb-hw                      # must fail (--rm)
+	test   -f tmp-tlb-hw.lz4
+	$(PRGDIR)/lz4cat tmp-tlb-hw.lz4           # must display hello world
+	test   -f tmp-tlb-hw.lz4
+	$(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
+	test   -f tmp-tlb-hw
+	test ! -f tmp-tlb-hw.lz4                  # must fail (--rm)
+	test ! -f tmp-tlb-hw.lz4.lz4              # must fail (unlz4)
+	$(PRGDIR)/lz4cat tmp-tlb-hw               # pass-through mode
+	test   -f tmp-tlb-hw
+	test ! -f tmp-tlb-hw.lz4                  # must fail (lz4cat)
+	$(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4          # creates tmp-tlb-hw.lz4
+	$(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3  # checks lz4cat works with stdin (#285)
+	$(DIFF) -q tmp-tlb-hw tmp-tlb3
+	$(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2      # checks lz4cat works in pass-through mode
+	$(DIFF) -q tmp-tlb-hw tmp-tlb2
+	cp tmp-tlb-hw ./-d
+	$(LZ4) --rm -- -d -d.lz4               # compresses ./d into ./-d.lz4
 	test   -f ./-d.lz4
 	test ! -f ./-d
 	mv ./-d.lz4 ./-z
-	$(LZ4) -d --rm -- -z tmp4          # uncompresses ./-z into tmp4
+	$(LZ4) -d --rm -- -z tmp-tlb4          # uncompresses ./-z into tmp-tlb4
 	test ! -f ./-z
-	$(DIFF) -q tmp tmp4
-	$(LZ4) -f tmp
-	cat tmp >> tmp.lz4
-	$(LZ4) -f tmp.lz4         # uncompress valid frame followed by invalid data
-	$(LZ4) -BX tmp -c -q | $(LZ4) -tv  # test block checksum
-	@$(RM) tmp*
+	$(DIFF) -q tmp-tlb-hw tmp-tlb4
+	$(LZ4) -f tmp-tlb-hw
+	cat tmp-tlb-hw >> tmp-tlb-hw.lz4
+	$(LZ4) -f tmp-tlb-hw.lz4               # uncompress valid frame followed by invalid data
+	$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv  # test block checksum
+	@$(RM) tmp-tlb*
 
 test-lz4-hugefile: lz4 datagen
 	@echo "\n ---- test huge files compression/decompression ----"
 	./datagen -g6GB   | $(LZ4) -vB5D  | $(LZ4) -qt
 	./datagen -g6GB   | $(LZ4) -v5BD  | $(LZ4) -qt
-	@$(RM) tmp*
 
 test-lz4-testmode: lz4 datagen
 	@echo "\n ---- bench mode ----"
-	  $(LZ4) -bi1
+	$(LZ4) -bi1
 	@echo "\n ---- test mode ----"
 	! ./datagen | $(LZ4) -t
 	! ./datagen | $(LZ4) -tf
 	@echo "\n ---- pass-through mode ----"
 	! ./datagen | $(LZ4) -d  > $(VOID)
-	  ./datagen | $(LZ4) -df > $(VOID)
-	@echo "Hello World !" > tmp1
-	$(LZ4) -dcf tmp1
-	@echo "from underground..." > tmp2
-	$(LZ4) -dcfm tmp1 tmp2
-	@echo "\n ---- test cli ----"
+	./datagen | $(LZ4) -df > $(VOID)
+	@echo "Hello World !" > tmp-tlt1
+	$(LZ4) -dcf tmp-tlt1
+	@echo "from underground..." > tmp-tlt2
+	$(LZ4) -dcfm tmp-tlt1 tmp-tlt2
+	@echo "\n ---- non-existing source ----"
 	! $(LZ4)     file-does-not-exist
 	! $(LZ4) -f  file-does-not-exist
 	! $(LZ4) -fm file1-dne file2-dne
-	! $(LZ4) -fm file1-dne file2-dne
+	@$(RM) tmp-tlt
 
 test-lz4-opt-parser: lz4 datagen
 	@echo "\n ---- test opt-parser ----"
@@ -304,9 +300,10 @@ test-lz4-opt-parser: lz4 datagen
 	./datagen -g16M -P90  | $(LZ4) -11B5    | $(LZ4) -t
 	./datagen -g32M -P10  | $(LZ4) -11B5D   | $(LZ4) -t
 
-test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \
-          test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
-          test-lz4-hugefile
+test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
+          test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \
+          test-lz4-contentSize test-lz4-hugefile
+	@$(RM) tmp*
 
 test-lz4c: lz4c datagen
 	@echo "\n ---- test lz4c version ----"
@@ -365,19 +362,19 @@ test-fasttest: fasttest
 test-mem: lz4 datagen fuzzer frametest fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
 	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
-	./datagen -g16KB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f tmp $(VOID)
-	./datagen -g16KB -s2 > tmp2
-	./datagen -g16KB -s3 > tmp3
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple tmp tmp2 tmp3
-	./datagen -g16MB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f tmp tmp2
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t tmp2
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 tmp
-	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2
-	./datagen -g256MB > tmp
-	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq tmp $(VOID)
-	$(RM) tmp*
+	./datagen -g16KB > ftmdg16K
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
+	./datagen -g16KB -s2 > ftmdg16K2
+	./datagen -g16KB -s3 > ftmdg16K3
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
+	./datagen -g16MB > ftmdg16M
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg16M ftmdg16K2
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg16M
+	valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg16M ftmdg16K2
+	./datagen -g256MB > ftmdg256M
+	valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg256M $(VOID)
+	$(RM) ftm*
 	valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
 	valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
 
-- 
cgit v0.12


From 68040a76ad834f6b600ad0d801a8f797855fa104 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 23 Aug 2017 17:53:45 -0700
Subject: fixed man directory installation (#337)

also : lz4c is now a symlink to lz4
---
 programs/Makefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index c7ef6d1..71b18b9 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -133,24 +133,25 @@ bindir      ?= $(exec_prefix)/bin
 BINDIR      ?= $(bindir)
 datarootdir ?= $(PREFIX)/share
 mandir      ?= $(datarootdir)/man
+man1dir     ?= $(mandir)/man1
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
 MANDIR  ?= $(PREFIX)/man/man1
 else
-MANDIR  ?= $(mandir)
+MANDIR  ?= $(man1dir)
 endif
 
 INSTALL_PROGRAM ?= $(INSTALL) -m 755
 INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
-install: lz4$(EXT) lz4c$(EXT)
+install: lz4
 	@echo Installing binaries
 	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
 	@$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4
+	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4c
 	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
 	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
-	@$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c
 	@echo Installing man pages
 	@$(INSTALL_DATA) lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
 	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
-- 
cgit v0.12


From 2ff723b9aedd9763a08a1f9641d5f9a8d8b20455 Mon Sep 17 00:00:00 2001
From: Markus Hennecke 
Date: Thu, 24 Aug 2017 14:31:45 +0200
Subject: Fix: Add return statement to main function

---
 examples/frameCompress.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 8712725..80db90e 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -309,4 +309,6 @@ int main(int argc, const char **argv) {
         fclose(decFp);
         fclose(inpFp);
     }
+
+    return 0;
 }
-- 
cgit v0.12


From f423d016cf76ac62551fb359afea26dfec675e8d Mon Sep 17 00:00:00 2001
From: tcpan 
Date: Thu, 24 Aug 2017 10:14:20 -0400
Subject: FIX: added prefix to FORCE_INLINE to prevent redefinition error
 during compilation when used with other libraries that define FORCE_INLINE

---
 lib/lz4.c    | 24 ++++++++++++------------
 lib/lz4hc.c  |  8 ++++----
 lib/lz4opt.h | 10 +++++-----
 lib/xxhash.c | 32 ++++++++++++++++----------------
 4 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 41c0a28..96422f8 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -101,21 +101,21 @@
 #  pragma warning(disable : 4293)        /* disable: C4293: too large shift (32-bits) */
 #endif  /* _MSC_VER */
 
-#ifndef FORCE_INLINE
+#ifndef LZ4_FORCE_INLINE
 #  ifdef _MSC_VER    /* Visual Studio */
-#    define FORCE_INLINE static __forceinline
+#    define LZ4_FORCE_INLINE static __forceinline
 #  else
 #    if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #      ifdef __GNUC__
-#        define FORCE_INLINE static inline __attribute__((always_inline))
+#        define LZ4_FORCE_INLINE static inline __attribute__((always_inline))
 #      else
-#        define FORCE_INLINE static inline
+#        define LZ4_FORCE_INLINE static inline
 #      endif
 #    else
-#      define FORCE_INLINE static
+#      define LZ4_FORCE_INLINE static
 #    endif /* __STDC_VERSION__ */
 #  endif  /* _MSC_VER */
-#endif /* FORCE_INLINE */
+#endif /* LZ4_FORCE_INLINE */
 
 #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
 #  define expect(expr,value)    (__builtin_expect ((expr),(value)) )
@@ -436,7 +436,7 @@ static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
         return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
 }
 
-FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
+LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
 {
     if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
     return LZ4_hash4(LZ4_read32(p), tableType);
@@ -452,7 +452,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy
     }
 }
 
-FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
@@ -465,7 +465,7 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tab
     { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; }   /* default, to ensure a return */
 }
 
-FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
 {
     U32 const h = LZ4_hashPosition(p, tableType);
     return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
@@ -474,7 +474,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy
 
 /** LZ4_compress_generic() :
     inlined, to ensure branches are decided at compilation time */
-FORCE_INLINE int LZ4_compress_generic(
+LZ4_FORCE_INLINE int LZ4_compress_generic(
                  LZ4_stream_t_internal* const cctx,
                  const char* const source,
                  char* const dest,
@@ -1105,7 +1105,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
  *  Note that it is important this generic function is really inlined,
  *  in order to remove useless branches during compilation optimization.
  */
-FORCE_INLINE int LZ4_decompress_generic(
+LZ4_FORCE_INLINE int LZ4_decompress_generic(
                  const char* const source,
                  char* const dest,
                  int inputSize,
@@ -1382,7 +1382,7 @@ Advanced decoding functions :
     the dictionary must be explicitly provided within parameters
 */
 
-FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
+LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
 {
     if (dictSize==0)
         return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 22eb071..d7f8d23 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -96,7 +96,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 
 
 /* Update chains up to ip (excluded) */
-FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
+LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const hashTable  = hc4->hashTable;
@@ -117,7 +117,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
 }
 
 
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
+LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                                const BYTE* const ip, const BYTE* const iLimit,
                                                const BYTE** matchpos,
                                                const int maxNbAttempts)
@@ -165,7 +165,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,
 }
 
 
-FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
+LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     LZ4HC_CCtx_internal* hc4,
     const BYTE* const ip,
     const BYTE* const iLowLimit,
@@ -248,7 +248,7 @@ typedef enum {
 /* LZ4HC_encodeSequence() :
  * @return : 0 if ok,
  *           1 if buffer issue detected */
-FORCE_INLINE int LZ4HC_encodeSequence (
+LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     const BYTE** ip,
     BYTE** op,
     const BYTE** anchor,
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index e9e54d8..584dc97 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -50,7 +50,7 @@ typedef struct {
 
 
 /* price in bytes */
-FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
+LZ4_FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 {
     size_t price = litlen;
     if (litlen >= (size_t)RUN_MASK)
@@ -60,7 +60,7 @@ FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
 
 
 /* requires mlen >= MINMATCH */
-FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
+LZ4_FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
 {
     size_t price = 2 + 1; /* 16-bit offset + token */
 
@@ -76,7 +76,7 @@ FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
 /*-*************************************
 *  Binary Tree search
 ***************************************/
-FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
+LZ4_FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
     LZ4HC_CCtx_internal* ctx,
     const BYTE* const ip,
     const BYTE* const iHighLimit,
@@ -170,7 +170,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
 }
 
 
-FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit)
+LZ4_FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit)
 {
     const BYTE* const base = ctx->base;
     const U32 target = (U32)(ip - base);
@@ -181,7 +181,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons
 
 
 /** Tree updater, providing best match */
-FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
+LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
                         LZ4HC_CCtx_internal* ctx,
                         const BYTE* const ip, const BYTE* const iHighLimit,
                         size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
diff --git a/lib/xxhash.c b/lib/xxhash.c
index a532358..bcf1f1d 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -115,21 +115,21 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
 #endif
 
-#ifndef FORCE_INLINE
+#ifndef XXH_FORCE_INLINE
 #  ifdef _MSC_VER    /* Visual Studio */
-#    define FORCE_INLINE static __forceinline
+#    define XXH_FORCE_INLINE static __forceinline
 #  else
 #    if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #      ifdef __GNUC__
-#        define FORCE_INLINE static inline __attribute__((always_inline))
+#        define XXH_FORCE_INLINE static inline __attribute__((always_inline))
 #      else
-#        define FORCE_INLINE static inline
+#        define XXH_FORCE_INLINE static inline
 #      endif
 #    else
-#      define FORCE_INLINE static
+#      define XXH_FORCE_INLINE static
 #    endif /* __STDC_VERSION__ */
 #  endif  /* _MSC_VER */
-#endif /* FORCE_INLINE */
+#endif /* XXH_FORCE_INLINE */
 
 
 /* *************************************
@@ -223,7 +223,7 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
 *****************************/
 typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
 
-FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+XXH_FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align==XXH_unaligned)
         return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
@@ -231,7 +231,7 @@ FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_a
         return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
 }
 
-FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+XXH_FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
 {
     return XXH_readLE32_align(ptr, endian, XXH_unaligned);
 }
@@ -266,7 +266,7 @@ static U32 XXH32_round(U32 seed, U32 input)
     return seed;
 }
 
-FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+XXH_FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* bEnd = p + len;
@@ -381,7 +381,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int s
 }
 
 
-FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
+XXH_FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
@@ -451,7 +451,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void*
 
 
 
-FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
+XXH_FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
 {
     const BYTE * p = (const BYTE*)state->mem32;
     const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
@@ -583,7 +583,7 @@ static U64 XXH_swap64 (U64 x)
 }
 #endif
 
-FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+XXH_FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align==XXH_unaligned)
         return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
@@ -591,7 +591,7 @@ FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_a
         return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
 }
 
-FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+XXH_FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
 {
     return XXH_readLE64_align(ptr, endian, XXH_unaligned);
 }
@@ -626,7 +626,7 @@ static U64 XXH64_mergeRound(U64 acc, U64 val)
     return acc;
 }
 
-FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+XXH_FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
@@ -750,7 +750,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long
     return XXH_OK;
 }
 
-FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
+XXH_FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
 {
     const BYTE* p = (const BYTE*)input;
     const BYTE* const bEnd = p + len;
@@ -815,7 +815,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void*
         return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
 }
 
-FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
+XXH_FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
 {
     const BYTE * p = (const BYTE*)state->mem64;
     const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
-- 
cgit v0.12


From 252ce14fd2ce8e4ff6038e79fe48a6b38643f8c9 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 24 Aug 2017 14:06:34 -0700
Subject: Don't use C++11 deprecation attribute with clang

Clang doesn't accept a C++11 attribute following `LZ4LIB_API`.
Use the GNU attribute instead.
---
 lib/lz4.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 86ca0d5..020a09d 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -419,9 +419,11 @@ union LZ4_streamDecode_u {
 #  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#  if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#  elif (LZ4_GCC_VERSION >= 405)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))
-- 
cgit v0.12


From 82ccdec2fb04316ae51ddf2666eecc397f2b1fd9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 24 Aug 2017 16:59:47 -0700
Subject: minor : added header license

---
 programs/Makefile |  2 +-
 tests/Makefile    |  3 +--
 tests/fasttest.c  | 36 ++++++++++++++++++++++++++++++------
 3 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 71b18b9..67af189 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -1,6 +1,6 @@
 # ##########################################################################
 # LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-2017
 #
 # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
 #
diff --git a/tests/Makefile b/tests/Makefile
index 5b30684..d2d05af 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,6 +1,6 @@
 # ##########################################################################
 # LZ4 programs - Makefile
-# Copyright (C) Yann Collet 2011-2016
+# Copyright (C) Yann Collet 2011-2017
 #
 # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
 #
@@ -76,7 +76,6 @@ lz4c:
 
 lz4c32:   # create a 32-bits version for 32/64 interop tests
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
-	cp $(LZ4) $(LZ4)c32
 
 fullbench  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
diff --git a/tests/fasttest.c b/tests/fasttest.c
index a165293..07811f6 100644
--- a/tests/fasttest.c
+++ b/tests/fasttest.c
@@ -1,8 +1,33 @@
+/*
+    fasttest.c -
+    Copyright (C) Yann Collet 2012-2017
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+    - LZ4 homepage : http://www.lz4.org
+    - LZ4 source repo : https://github.com/lz4/lz4
+*/
+
 /**************************************
  * Compiler Options
  **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS // for MSVC
+#  define _CRT_SECURE_NO_WARNINGS  /* for MSVC */
 #  define snprintf sprintf_s
 #endif
 #ifdef __GNUC__
@@ -35,16 +60,15 @@ int test_compress(const char *input, int inSize, char *output, int outSize)
         const int length = inSize >> 2;
         if (inSize > 1024) return -2;
         if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3;
-        {
-            const int outBytes = LZ4_compress_fast_continue(
-                lz4Stream, input + inOffset, output + outOffset + 8, length, outSize-outOffset, 1);
+        {   const int outBytes = LZ4_compress_fast_continue(
+                lz4Stream, input + inOffset, output + outOffset + 8,
+                length, outSize-outOffset, 1);
             if(outBytes <= 0) return -4;
             memcpy(output + outOffset, &length, 4); /* input length */
             memcpy(output + outOffset + 4, &outBytes, 4); /* output length */
             inOffset += length;
             outOffset += outBytes + 8;
-        }
-    }
+    }   }
     if (outOffset + 8 > outSize) return -5;
     memset(output + outOffset, 0, 4);
     memset(output + outOffset + 4, 0, 4);
-- 
cgit v0.12


From 517dd95d92980cb709f8ad176ca17c2b7e79e709 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 24 Aug 2017 17:03:03 -0700
Subject: removed fasttest

---
 tests/Makefile   |  10 +---
 tests/fasttest.c | 155 -------------------------------------------------------
 2 files changed, 2 insertions(+), 163 deletions(-)
 delete mode 100644 tests/fasttest.c

diff --git a/tests/Makefile b/tests/Makefile
index d2d05af..fc4b143 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -63,7 +63,7 @@ NB_LOOPS     ?= -i1
 
 default: all
 
-all: fullbench fuzzer frametest datagen fasttest
+all: fullbench fuzzer frametest datagen
 
 all32: CFLAGS+=-m32
 all32: all
@@ -94,9 +94,6 @@ fuzzer  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
 frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-fasttest: $(LZ4DIR)/lz4.o fasttest.c
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
-
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
 
@@ -133,7 +130,7 @@ DIFF:=gdiff
 endif
 
 
-test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer
 
 test32: CFLAGS+=-m32
 test32: test
@@ -355,9 +352,6 @@ test-frametest: frametest
 test-frametest32: CFLAGS += -m32
 test-frametest32: test-frametest
 
-test-fasttest: fasttest
-	./fasttest
-
 test-mem: lz4 datagen fuzzer frametest fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
 	valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
diff --git a/tests/fasttest.c b/tests/fasttest.c
deleted file mode 100644
index 07811f6..0000000
--- a/tests/fasttest.c
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
-    fasttest.c -
-    Copyright (C) Yann Collet 2012-2017
-
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - LZ4 homepage : http://www.lz4.org
-    - LZ4 source repo : https://github.com/lz4/lz4
-*/
-
-/**************************************
- * Compiler Options
- **************************************/
-#ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS  /* for MSVC */
-#  define snprintf sprintf_s
-#endif
-#ifdef __GNUC__
-#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
-
-
-/**************************************
- * Includes
- **************************************/
-#include 
-#include 
-#include 
-#include 
-#include "lz4.h"
-
-
-/* Returns non-zero on failure. */
-int test_compress(const char *input, int inSize, char *output, int outSize)
-{
-    LZ4_stream_t lz4Stream_body = { 0 };
-    LZ4_stream_t* lz4Stream = &lz4Stream_body;
-
-    int inOffset = 0;
-    int outOffset = 0;
-
-    if (inSize & 3) return -1;
-
-    while (inOffset < inSize) {
-        const int length = inSize >> 2;
-        if (inSize > 1024) return -2;
-        if (outSize - (outOffset + 8) < LZ4_compressBound(length)) return -3;
-        {   const int outBytes = LZ4_compress_fast_continue(
-                lz4Stream, input + inOffset, output + outOffset + 8,
-                length, outSize-outOffset, 1);
-            if(outBytes <= 0) return -4;
-            memcpy(output + outOffset, &length, 4); /* input length */
-            memcpy(output + outOffset + 4, &outBytes, 4); /* output length */
-            inOffset += length;
-            outOffset += outBytes + 8;
-    }   }
-    if (outOffset + 8 > outSize) return -5;
-    memset(output + outOffset, 0, 4);
-    memset(output + outOffset + 4, 0, 4);
-    return 0;
-}
-
-/* Returns non-zero on failure. Not a safe function. */
-int test_decompress(const char *uncompressed, const char *compressed)
-{
-    char outBufferA[1024];
-    char spacing; /* So prefixEnd != dest */
-    char outBufferB[1024];
-    char *output = outBufferA;
-    char *lastOutput = outBufferB;
-    LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
-    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
-    int offset = 0;
-    int unOffset = 0;
-    int lastBytes = 0;
-
-    (void)spacing;
-
-    for(;;) {
-        int32_t bytes;
-        int32_t unBytes;
-        /* Read uncompressed size and compressed size */
-        memcpy(&unBytes, compressed + offset, 4);
-        memcpy(&bytes, compressed + offset + 4, 4);
-        offset += 8;
-        /* Check if we reached end of stream or error */
-        if(bytes == 0 && unBytes == 0) return 0;
-        if(bytes <= 0 || unBytes <= 0 || unBytes > 1024) return 1;
-
-        /* Put the last output in the dictionary */
-        LZ4_setStreamDecode(lz4StreamDecode, lastOutput, lastBytes);
-        /* Decompress */
-        bytes = LZ4_decompress_fast_continue(
-            lz4StreamDecode, compressed + offset, output, unBytes);
-        if(bytes <= 0) return 2;
-        /* Check result */
-        {   int const r = memcmp(uncompressed + unOffset, output, unBytes);
-            if (r) return 3;
-        }
-        { char* const tmp = output; output = lastOutput; lastOutput = tmp; }
-        offset += bytes;
-        unOffset += unBytes;
-        lastBytes = unBytes;
-    }
-}
-
-
-int main(int argc, char **argv)
-{
-    char input[] =
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello!"
-        "Hello Hello Hello Hello Hello Hello Hello Hello";
-    char output[LZ4_COMPRESSBOUND(4096)];
-    int r;
-
-    (void)argc;
-    (void)argv;
-
-    if ((r = test_compress(input, sizeof(input), output, sizeof(output)))) {
-        return r;
-    }
-    if ((r = test_decompress(input, output))) {
-        return r;
-    }
-    return 0;
-}
-- 
cgit v0.12


From c04655bebf0990bc3c14e1cda122b992733b293d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 24 Aug 2017 17:58:13 -0700
Subject: fixed make recurrence from /tests

---
 programs/Makefile |  9 +++++----
 tests/Makefile    | 12 +++---------
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 67af189..29408f1 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,7 +42,8 @@ LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER   := $(shell echo $(LIBVER_SCRIPT))
 
 SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)
-OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
+#OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
+OBJFILES := $(SRCFILES:.c=.o)
 
 CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 CFLAGS   ?= -O3
@@ -81,13 +82,13 @@ lz4: $(OBJFILES)
 lz4-release: DEBUGFLAGS=
 lz4-release: lz4
 
+lz4c: lz4
+	ln -s lz4 lz4c
+
 lz4c32: CFLAGS += -m32
 lz4c32 : $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lz4c: lz4
-	ln -s lz4 lz4c
-
 lz4.1: lz4.1.md
 	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 
diff --git a/tests/Makefile b/tests/Makefile
index fc4b143..bc13295 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -71,8 +71,8 @@ all32: all
 lz4:
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
 
-lz4c:
-	$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
+lz4c unlz4 lz4cat: lz4
+	ln -sf $(LZ4) $(PRGDIR)/$@
 
 lz4c32:   # create a 32-bits version for 32/64 interop tests
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
@@ -115,7 +115,7 @@ versionsTest:
 
 
 #-----------------------------------------------------------------------------
-# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+# validated only for Linux, OSX, BSD, Hurd and Solaris targets
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
@@ -207,12 +207,6 @@ test-lz4-multiple: lz4 datagen
 	$(LZ4) -f -m tmp-tlm1 notHere tmp-tlm2; echo $$?
 	@$(RM) tmp-tlm*
 
-unlz4:
-	@$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
-
-lz4cat:
-	@$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
-
 test-lz4-basic: lz4 datagen unlz4 lz4cat
 	@echo "\n ---- test lz4 basic compression/decompression ----"
 	./datagen -g0       | $(LZ4) -v     | $(LZ4) -t
-- 
cgit v0.12


From 2ed36124778adfc5abd43e875c45fc310a52ee5d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 24 Aug 2017 18:30:00 -0700
Subject: removed test-fasttest

---
 .travis.yml    | 2 +-
 tests/Makefile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dc61505..2724e92 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,7 @@ matrix:
     # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
-      env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fasttest test-fullbench' COMPILER=cc
+      env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc
 
     - os: linux
       sudo: false
diff --git a/tests/Makefile b/tests/Makefile
index bc13295..e870fcf 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -296,7 +296,7 @@ test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
 	@$(RM) tmp*
 
 test-lz4c: lz4c datagen
-	@echo "\n ---- test lz4c version ----"
+	@echo "\n ---- test lz4c variant ----"
 	./datagen -g256MB | $(LZ4)c -l -v    | $(LZ4)c   -t
 
 test-lz4c32: CFLAGS+=-m32
-- 
cgit v0.12


From a261f51ee31bbbb18d0df9401a7f93bafd49b14c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Aug 2017 09:50:56 -0700
Subject: removed fasttest from circleCI

---
 circle.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/circle.yml b/circle.yml
index b3faae0..9a0d1ec 100644
--- a/circle.yml
+++ b/circle.yml
@@ -19,7 +19,6 @@ test:
     - make cmake               && make clean
     - make -C tests test-lz4
     - make -C tests test-lz4c
-    - make -C tests test-fasttest
     - make -C tests test-frametest
     - make -C tests test-fullbench
     - make -C tests test-fuzzer && make clean
-- 
cgit v0.12


From d2306b836e8055ee14e0a239ad8ef44dc5100374 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 25 Aug 2017 14:13:29 -0700
Subject: updated NEWS with fixes

---
 NEWS | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/NEWS b/NEWS
index e63a82f..57338f7 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+v1.8.1
+install: fix : correct man page directory (#387), reported by Stuart Cardall (@itoffshore)
+build  : `make` and `make test` compatible with `-jX`, reported by @mwgamera
+
 v1.8.0
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
 cli : added GNU separator -- specifying that all following arguments are files
-- 
cgit v0.12


From c4d6884cd82fddbbdce015750e574fd02af36a94 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 26 Aug 2017 12:22:51 -0700
Subject: build: source files sorted in a FS independent manner

to be more compatible with reproducible builds.
patch inspired by @bmwiedemann
---
 doc/lz4_manual.html | 6 ++++--
 lib/Makefile        | 5 +++--
 programs/Makefile   | 6 +++---
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 9ab1984..df8128b 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -301,9 +301,11 @@ union LZ4_streamDecode_u {
 #  define LZ4_DEPRECATED(message)   /* disable deprecation warnings */
 #else
 #  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#  if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
+#  if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
 #    define LZ4_DEPRECATED(message) [[deprecated(message)]]
-#  elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#  elif (LZ4_GCC_VERSION >= 405)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
 #  elif (LZ4_GCC_VERSION >= 301)
 #    define LZ4_DEPRECATED(message) __attribute__((deprecated))
diff --git a/lib/Makefile b/lib/Makefile
index 9abb699..ed546dd 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -47,11 +47,12 @@ BUILD_STATIC:=yes
 CPPFLAGS+= -DXXH_NAMESPACE=LZ4_
 CFLAGS  ?= -O3
 DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-           -Wpointer-arith -Wstrict-aliasing=1
+             -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
+             -Wundef -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
+SRCFILES := $(sort $(wildcard *.c))
 
 
 # OS X linker doesn't support -soname, and use different extension
diff --git a/programs/Makefile b/programs/Makefile
index 29408f1..a551b01 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -41,15 +41,15 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
 LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER   := $(shell echo $(LIBVER_SCRIPT))
 
-SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c)
+SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c))
 #OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
 OBJFILES := $(SRCFILES:.c=.o)
 
 CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
 CFLAGS   ?= -O3
 DEBUGFLAGS:=-Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-           -Wpointer-arith -Wstrict-aliasing=1
+            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
+            -Wpointer-arith -Wstrict-aliasing=1
 CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS     = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
-- 
cgit v0.12


From f212a10ae01f8bd1fe33d36ef51574a7e3215a02 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 29 Aug 2017 15:31:56 -0700
Subject: fixed FS-independent file order in /lib

identified by @bmwiedemann
---
 lib/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index ed546dd..5907aac 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -84,14 +84,14 @@ all: lib
 all32: CFLAGS+=-m32
 all32: all
 
-liblz4.a: *.c
+liblz4.a: $(SRCFILES)
 ifeq ($(BUILD_STATIC),yes)  # can be disabled on command line
 	@echo compiling static library
 	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
 	@$(AR) rcs $@ *.o
 endif
 
-$(LIBLZ4): *.c
+$(LIBLZ4): $(SRCFILES)
 	@echo compiling dynamic library $(LIBVER)
 ifneq (,$(filter Windows%,$(OS)))
 	@$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll
-- 
cgit v0.12


From 69c3f4bb55c6f0498f3ba6e23218e82c5d6ab075 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 30 Aug 2017 15:02:04 -0700
Subject: minor typo fix

---
 INSTALL   | 1 +
 README.md | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/INSTALL b/INSTALL
index f5ac0a0..6aab067 100644
--- a/INSTALL
+++ b/INSTALL
@@ -8,6 +8,7 @@ make install     # this command may require root access
 
 LZ4's `Makefile` supports standard [Makefile conventions],
 including [staged installs], [redirection], or [command redefinition].
+It is compatible with parallel builds (`-j#`).
 
 [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
diff --git a/README.md b/README.md
index d7e8fb6..596fdac 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ make install     # this command may require root access
 
 LZ4's `Makefile` supports standard [Makefile conventions],
 including [staged installs], [redirection], or [command redefinition].
-It is compatible with parallel builds (`'-j#'`).
+It is compatible with parallel builds (`-j#`).
 
 [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html
-- 
cgit v0.12


From e3c71584ff3242ce0a42e1cdeb183756b2c47da4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 30 Aug 2017 15:42:04 -0700
Subject: clarified documentation of streaming decompression functions

(synchronous bufferless mode)
answering questions by @jtbandes (#394)
---
 doc/lz4_manual.html | 22 +++++++++++++---------
 lib/lz4.h           | 22 +++++++++++++---------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index df8128b..e3f6681 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -193,32 +193,36 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
 
LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-
  creation / destruction of streaming decompression tracking structure 
+
  creation / destruction of streaming decompression tracking structure.
+  A tracking structure can be re-used multiple times sequentially. 
 


 
 
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-
  Use this function to instruct where to find the dictionary.
-  Setting a size of 0 is allowed (same effect as reset).
-  @return : 1 if OK, 0 if error
+
  Use this function to start decompression of a new stream of blocks.
+  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+ @return : 1 if OK, 0 if error
  
 


 
 
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-
  These decoding functions allow decompression of multiple blocks in "streaming" mode.
-  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
-  In the case of a ring buffers, decoding buffer must be either :
+
  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
+  A block is an unsplittable entity, it must be presented entirely to a decompression function.
+  Decompression functions only accept one block at a time.
+  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+
+  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
   - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
     In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
   - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
-    maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+    maxBlockSize is implementation dependent. It's the maximum size of any single block.
     In which case, encoding and decoding buffers do not need to be synchronized,
     and encoding ring buffer can have any size, including small ones ( < 64 KB).
   - _At least_ 64 KB + 8 bytes + maxBlockSize.
     In which case, encoding and decoding buffers do not need to be synchronized,
     and encoding ring buffer can have any size, including larger than decoding buffer.
   Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-  and indicate where it is saved using LZ4_setStreamDecode()
+  and indicate where it is saved using LZ4_setStreamDecode() before decompressin next block.
 


 
 
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
diff --git a/lib/lz4.h b/lib/lz4.h
index 020a09d..3564a1d 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -280,32 +280,36 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict
 typedef union LZ4_streamDecode_u LZ4_streamDecode_t;   /* incomplete type (defined later) */
 
 /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
- *  creation / destruction of streaming decompression tracking structure */
+ *  creation / destruction of streaming decompression tracking structure.
+ *  A tracking structure can be re-used multiple times sequentially. */
 LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
 /*! LZ4_setStreamDecode() :
- *  Use this function to instruct where to find the dictionary.
- *  Setting a size of 0 is allowed (same effect as reset).
- *  @return : 1 if OK, 0 if error
+ *  Use this function to start decompression of a new stream of blocks.
+ *  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
+ * @return : 1 if OK, 0 if error
  */
 LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
 
 /*! LZ4_decompress_*_continue() :
- *  These decoding functions allow decompression of multiple blocks in "streaming" mode.
- *  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- *  In the case of a ring buffers, decoding buffer must be either :
+ *  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
+ *  A block is an unsplittable entity, it must be presented entirely to a decompression function.
+ *  Decompression functions only accept one block at a time.
+ *  Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB).
+ *
+ *  Special : if application sets a ring buffer for decompression, it must respect one of the following conditions :
  *  - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
  *    In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
  *  - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- *    maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ *    maxBlockSize is implementation dependent. It's the maximum size of any single block.
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including small ones ( < 64 KB).
  *  - _At least_ 64 KB + 8 bytes + maxBlockSize.
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including larger than decoding buffer.
  *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- *  and indicate where it is saved using LZ4_setStreamDecode()
+ *  and indicate where it is saved using LZ4_setStreamDecode() before decompressin next block.
 */
 LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
 LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-- 
cgit v0.12


From 3d260f352293d5cddc8e7f7464b93308d47ed176 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 30 Aug 2017 16:09:10 -0700
Subject: complementary information for #394

---
 doc/lz4_manual.html | 24 +++++++++++++++---------
 lib/lz4.h           | 22 ++++++++++++++--------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index e3f6681..32d98e2 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -158,24 +158,29 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
 
void LZ4_resetStream (LZ4_stream_t* streamPtr);
 
  An LZ4_stream_t structure can be allocated once and re-used multiple times.
-  Use this function to init an allocated `LZ4_stream_t` structure and start a new compression.
+  Use this function to start compressing a new stream.
  
 


 
 
int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
-
  Use this function to load a static dictionary into LZ4_stream.
+
  Use this function to load a static dictionary into LZ4_stream_t.
   Any previous data will be forgotten, only 'dictionary' will remain in memory.
-  Loading a size of 0 is allowed.
-  Return : dictionary size, in bytes (necessarily <= 64 KB)
+  Loading a size of 0 is allowed, and is the same as reset.
+ @return : dictionary size, in bytes (necessarily <= 64 KB)
  
 


 
 
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
-
  Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
-  Important : Previous data blocks are assumed to remain present and unmodified !
+
  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
   'dst' buffer must be already allocated.
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
-  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+
+  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+
+ @return : size of compressed block
+           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
   After an error, the stream status is invalid, it can only be reset or freed.
  
 


@@ -198,7 +203,8 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 


 
 
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-
  Use this function to start decompression of a new stream of blocks.
+
  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
+  Use this function to start decompression of a new stream of blocks.
   A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
  @return : 1 if OK, 0 if error
  
@@ -222,7 +228,7 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     In which case, encoding and decoding buffers do not need to be synchronized,
     and encoding ring buffer can have any size, including larger than decoding buffer.
   Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
-  and indicate where it is saved using LZ4_setStreamDecode() before decompressin next block.
+  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 


 
 
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
diff --git a/lib/lz4.h b/lib/lz4.h
index 3564a1d..d284d63 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -242,24 +242,29 @@ LZ4LIB_API int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
 /*! LZ4_resetStream() :
  *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
- *  Use this function to init an allocated `LZ4_stream_t` structure and start a new compression.
+ *  Use this function to start compressing a new stream.
  */
 LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
 
 /*! LZ4_loadDict() :
- *  Use this function to load a static dictionary into LZ4_stream.
+ *  Use this function to load a static dictionary into LZ4_stream_t.
  *  Any previous data will be forgotten, only 'dictionary' will remain in memory.
- *  Loading a size of 0 is allowed.
- *  Return : dictionary size, in bytes (necessarily <= 64 KB)
+ *  Loading a size of 0 is allowed, and is the same as reset.
+ * @return : dictionary size, in bytes (necessarily <= 64 KB)
  */
 LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
 
 /*! LZ4_compress_fast_continue() :
- *  Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- *  Important : Previous data blocks are assumed to remain present and unmodified !
+ *  Compress content into 'src' using data from previously compressed blocks, improving compression ratio.
  *  'dst' buffer must be already allocated.
  *  If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- *  If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function @return==0.
+ *
+ *  Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
+ *  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+ *  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+ *
+ * @return : size of compressed block
+ *           or 0 if there is an error (typically, compressed data cannot fit into 'dst')
  *  After an error, the stream status is invalid, it can only be reset or freed.
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -286,6 +291,7 @@ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
 LZ4LIB_API int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
 
 /*! LZ4_setStreamDecode() :
+ *  An LZ4_streamDecode_t structure can be allocated once and re-used multiple times.
  *  Use this function to start decompression of a new stream of blocks.
  *  A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order.
  * @return : 1 if OK, 0 if error
@@ -309,7 +315,7 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *    In which case, encoding and decoding buffers do not need to be synchronized,
  *    and encoding ring buffer can have any size, including larger than decoding buffer.
  *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- *  and indicate where it is saved using LZ4_setStreamDecode() before decompressin next block.
+ *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 */
 LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
 LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-- 
cgit v0.12


From 1ad9a36cfb9fc669b0b18bfcd902d3e52a3d2cbb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 6 Sep 2017 11:22:45 -0700
Subject: updated lib/README

clarifications, improved wording
---
 lib/README.md | 56 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/lib/README.md b/lib/README.md
index b40442c..7082fe3 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -1,44 +1,43 @@
 LZ4 - Library Files
 ================================
 
-The directory contains many files, but depending on project's objectives,
+The `/lib` directory contains many files, but depending on project's objectives,
 not all of them are necessary.
 
 #### Minimal LZ4 build
 
 The minimum required is **`lz4.c`** and **`lz4.h`**,
-which will provide the fast compression and decompression algorithm.
+which provides the fast compression and decompression algorithm.
+They generate and decode data using [LZ4 block format].
 
 
-#### The High Compression variant of LZ4
+#### High Compression variant
 
-For more compression at the cost of compression speed,
-the High Compression variant **lz4hc** is available.
-It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**.
-The variant still depends on regular `lz4` source files.
-In particular, the decompression is still provided by `lz4.c`.
+For more compression ratio at the cost of compression speed,
+the High Compression variant called **lz4hc** is available.
+Add files **`lz4hc.c`** and **`lz4hc.h`**.
+The variant still depends on regular `lib/lz4.*` source files.
 
 
-#### Compatibility issues
+#### Frame variant, for interoperability
 
-In order to produce files or streams compatible with `lz4` command line utility,
+In order to produce compressed data compatible with `lz4` command line utility,
 it's necessary to encode lz4-compressed blocks using the [official interoperable frame format].
 This format is generated and decoded automatically by the **lz4frame** library.
-In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**,
-which provides error detection.
-(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace.
-This is what `liblz4` does, to avoid symbol duplication
-in case a user program would link to several libraries containing xxhash symbols.)
+Its public API is described in `lib/lz4frame.h`.
+In order to work properly, lz4frame needs all other modules present in `/lib`,
+including, lz4 and lz4hc, and also **xxhash**.
+So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
 
 
-#### Advanced API 
+#### Advanced / Experimental API
 
-A more complex `lz4frame_static.h` is also provided.
-It contains definitions which are not guaranteed to remain stable within future versions.
-It must be used with static linking ***only***.
+A complex API defined in `lz4frame_static.h` contains definitions
+which are not guaranteed to remain stable in future versions.
+As a consequence, it must be used with static linking ***only***.
 
 
-#### Using MinGW+MSYS to create DLL
+#### Windows : using MinGW+MSYS to create DLL
 
 DLL can be created using MinGW+MSYS with the `make liblz4` command.
 This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`.
@@ -51,23 +50,24 @@ file it should be linked with `dll\liblz4.dll`. For example:
 ```
     gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll
 ```
-The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. 
+The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`.
 
 
-#### Miscellaneous 
+#### Miscellaneous
 
 Other files present in the directory are not source code. There are :
 
- - LICENSE : contains the BSD license text
- - Makefile : script to compile or install lz4 library (static or dynamic)
- - liblz4.pc.in : for pkg-config (make install)
- - README.md : this file
+ - `LICENSE` : contains the BSD license text
+ - `Makefile` : `make` script to compile and install lz4 library (static and dynamic)
+ - `liblz4.pc.in` : for `pkg-config` (used in `make install`)
+ - `README.md` : this file
 
 [official interoperable frame format]: ../doc/lz4_Frame_format.md
+[LZ4 block format]: ../doc/lz4_Block_format.md
 
 
-#### License 
+#### License
 
 All source material within __lib__ directory are BSD 2-Clause licensed.
 See [LICENSE](LICENSE) for details.
-The license is also repeated at the top of each source file.
+The license is also reminded at the top of each source file.
-- 
cgit v0.12


From ee5332438b9a4c65147003334bcdbb9915930603 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 6 Sep 2017 17:41:44 -0700
Subject: minor Makefile fixes

---
 lib/Makefile      |  3 ++-
 programs/Makefile | 16 ++++++++--------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index 5907aac..dd33f50 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -117,7 +117,8 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
 DESTDIR     ?=
-# directory variables : GNU convention prefers lowercase
+# directory variables : GNU conventions prefer lowercase
+# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 # support both lower and uppercase (BSD), use uppercase in script
 prefix      ?= /usr/local
 PREFIX      ?= $(prefix)
diff --git a/programs/Makefile b/programs/Makefile
index a551b01..ed1a779 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,7 +42,6 @@ LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
 LIBVER   := $(shell echo $(LIBVER_SCRIPT))
 
 SRCFILES := $(sort $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c))
-#OBJFILES := $(patsubst %.c,%.o,$(SRCFILES))
 OBJFILES := $(SRCFILES:.c=.o)
 
 CPPFLAGS += -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_
@@ -118,14 +117,9 @@ unlz4: lz4
 lz4cat: lz4
 	ln -s lz4 lz4cat
 
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
 DESTDIR     ?=
-# directory variables : GNU convention prefers lowercase
+# directory variables : GNU conventions prefer lowercase
+# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 # support both lower and uppercase (BSD), use uppercase in script
 prefix      ?= /usr/local
 PREFIX      ?= $(prefix)
@@ -142,6 +136,12 @@ else
 MANDIR  ?= $(man1dir)
 endif
 
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
 INSTALL_PROGRAM ?= $(INSTALL) -m 755
 INSTALL_DATA    ?= $(INSTALL) -m 644
 
-- 
cgit v0.12


From 00418f76115e3c435b4cb0e41b30fba8e8e3adee Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 7 Sep 2017 12:12:36 -0700
Subject: bench : made decompression speed evaluation same time as compression

minor : slightly modified an example do avoid disabling a gcc warning through #pragma
---
 examples/blockStreaming_ringBuffer.c | 45 ++++++++++++++----------------------
 lib/lz4.c                            | 33 +++++++++++++-------------
 programs/bench.c                     |  6 ++---
 3 files changed, 36 insertions(+), 48 deletions(-)

diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c
index 697d342..40109b5 100644
--- a/examples/blockStreaming_ringBuffer.c
+++ b/examples/blockStreaming_ringBuffer.c
@@ -1,5 +1,5 @@
-// LZ4 streaming API example : ring buffer
-// Based on sample code from Takayuki Matsuoka
+/* LZ4 streaming API example : ring buffer
+ * Based on sample code from Takayuki Matsuoka */
 
 
 /**************************************
@@ -9,9 +9,6 @@
 #  define _CRT_SECURE_NO_WARNINGS // for MSVC
 #  define snprintf sprintf_s
 #endif
-#ifdef __GNUC__
-#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
 
 
 /**************************************
@@ -50,7 +47,7 @@ size_t read_bin(FILE* fp, void* array, int arrayBytes) {
 
 void test_compress(FILE* outFp, FILE* inpFp)
 {
-    LZ4_stream_t lz4Stream_body = { 0 };
+    LZ4_stream_t lz4Stream_body = { { 0 } };
     LZ4_stream_t* lz4Stream = &lz4Stream_body;
 
     static char inpBuf[RING_BUFFER_BYTES];
@@ -85,24 +82,22 @@ void test_compress(FILE* outFp, FILE* inpFp)
 void test_decompress(FILE* outFp, FILE* inpFp)
 {
     static char decBuf[DECODE_RING_BUFFER];
-    int   decOffset    = 0;
-    LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
+    int decOffset = 0;
+    LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } };
     LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
 
     for(;;) {
         int cmpBytes = 0;
         char cmpBuf[CMPBUFSIZE];
 
-        {
-            const size_t r0 = read_int32(inpFp, &cmpBytes);
+        {   const size_t r0 = read_int32(inpFp, &cmpBytes);
             if(r0 != 1 || cmpBytes <= 0) break;
 
             const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
             if(r1 != (size_t) cmpBytes) break;
         }
 
-        {
-            char* const decPtr = &decBuf[decOffset];
+        {   char* const decPtr = &decBuf[decOffset];
             const int decBytes = LZ4_decompress_safe_continue(
                 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
             if(decBytes <= 0) break;
@@ -120,7 +115,7 @@ int compare(FILE* f0, FILE* f1)
 {
     int result = 0;
 
-    while(0 == result) {
+    while (0 == result) {
         char b0[65536];
         char b1[65536];
         const size_t r0 = fread(b0, 1, sizeof(b0), f0);
@@ -128,12 +123,9 @@ int compare(FILE* f0, FILE* f1)
 
         result = (int) r0 - (int) r1;
 
-        if(0 == r0 || 0 == r1) {
-            break;
-        }
-        if(0 == result) {
-            result = memcmp(b0, b1, r0);
-        }
+        if (0 == r0 || 0 == r1) break;
+
+        if (0 == result) result = memcmp(b0, b1, r0);
     }
 
     return result;
@@ -160,9 +152,8 @@ int main(int argc, char** argv)
     printf("dec = [%s]\n", decFilename);
 
     // compress
-    {
-        FILE* inpFp = fopen(inpFilename, "rb");
-        FILE* outFp = fopen(lz4Filename, "wb");
+    {   FILE* const inpFp = fopen(inpFilename, "rb");
+        FILE* const outFp = fopen(lz4Filename, "wb");
 
         test_compress(outFp, inpFp);
 
@@ -171,9 +162,8 @@ int main(int argc, char** argv)
     }
 
     // decompress
-    {
-        FILE* inpFp = fopen(lz4Filename, "rb");
-        FILE* outFp = fopen(decFilename, "wb");
+    {   FILE* const inpFp = fopen(lz4Filename, "rb");
+        FILE* const outFp = fopen(decFilename, "wb");
 
         test_decompress(outFp, inpFp);
 
@@ -182,9 +172,8 @@ int main(int argc, char** argv)
     }
 
     // verify
-    {
-        FILE* inpFp = fopen(inpFilename, "rb");
-        FILE* decFp = fopen(decFilename, "rb");
+    {   FILE* const inpFp = fopen(inpFilename, "rb");
+        FILE* const decFp = fopen(decFilename, "rb");
 
         const int cmp = compare(inpFp, decFp);
         if(0 == cmp) {
diff --git a/lib/lz4.c b/lib/lz4.c
index 96422f8..f361b22 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -289,7 +289,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
 /*-************************************
 *  Error detection
 **************************************/
-#define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+#define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
 #  include 
@@ -1100,31 +1100,30 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 *  Decompression functions
 *******************************/
 /*! LZ4_decompress_generic() :
- *  This generic decompression function cover all use cases.
- *  It shall be instantiated several times, using different sets of directives
- *  Note that it is important this generic function is really inlined,
+ *  This generic decompression function covers all use cases.
+ *  It shall be instantiated several times, using different sets of directives.
+ *  Note that it is important for performance that this function really get inlined,
  *  in order to remove useless branches during compilation optimization.
  */
 LZ4_FORCE_INLINE int LZ4_decompress_generic(
-                 const char* const source,
-                 char* const dest,
-                 int inputSize,
-                 int outputSize,         /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
+                 const char* const src,
+                 char* const dst,
+                 int srcSize,
+                 int outputSize,         /* If endOnInput==endOnInputSize, this value is `dstCapacity` */
 
                  int endOnInput,         /* endOnOutputSize, endOnInputSize */
                  int partialDecoding,    /* full, partial */
                  int targetOutputSize,   /* only used if partialDecoding==partial */
                  int dict,               /* noDict, withPrefix64k, usingExtDict */
-                 const BYTE* const lowPrefix,  /* == dest when no prefix */
+                 const BYTE* const lowPrefix,  /* == dst when no prefix */
                  const BYTE* const dictStart,  /* only if dict==usingExtDict */
                  const size_t dictSize         /* note : = 0 if noDict */
                  )
 {
-    /* Local Variables */
-    const BYTE* ip = (const BYTE*) source;
-    const BYTE* const iend = ip + inputSize;
+    const BYTE* ip = (const BYTE*) src;
+    const BYTE* const iend = ip + srcSize;
 
-    BYTE* op = (BYTE*) dest;
+    BYTE* op = (BYTE*) dst;
     BYTE* const oend = op + outputSize;
     BYTE* cpy;
     BYTE* oexit = op + targetOutputSize;
@@ -1140,7 +1139,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
     /* Special cases */
     if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                        /* targetOutputSize too high => decode everything */
-    if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
+    if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
     if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
 
     /* Main Loop : decode sequences */
@@ -1257,13 +1256,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
     /* end of decoding */
     if (endOnInput)
-       return (int) (((char*)op)-dest);     /* Nb of output bytes decoded */
+       return (int) (((char*)op)-dst);     /* Nb of output bytes decoded */
     else
-       return (int) (((const char*)ip)-source);   /* Nb of input bytes read */
+       return (int) (((const char*)ip)-src);   /* Nb of input bytes read */
 
     /* Overflow error detected */
 _output_error:
-    return (int) (-(((const char*)ip)-source))-1;
+    return (int) (-(((const char*)ip)-src))-1;
 }
 
 
diff --git a/programs/bench.c b/programs/bench.c
index 77a9e3f..05ddaff 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -68,7 +68,7 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
 #define COOLPERIOD_SEC        10
-#define DECOMP_MULT           2 /* test decompression DECOMP_MULT times longer than compression */
+#define DECOMP_MULT           1 /* test decompression DECOMP_MULT times longer than compression */
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -456,9 +456,9 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
     if (benchedSize==0) EXM_THROW(12, "not enough memory");
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize > LZ4_MAX_INPUT_SIZE) {
-        benchedSize = LZ4_MAX_INPUT_SIZE; 
+        benchedSize = LZ4_MAX_INPUT_SIZE;
         DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20));
-    } else { 
+    } else {
         if (benchedSize < totalSizeToLoad)
             DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
     }
-- 
cgit v0.12


From b500c5282e61027b797d3fd39e7d4ab1ab084adf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 7 Sep 2017 12:48:24 -0700
Subject: fix #397 : decompression failed when using a combination of extDict +
 low memory address

Reported and fixed by @jscheid

Note : we are missing a test case to include it in the CI
---
 NEWS      | 1 +
 lib/lz4.c | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 57338f7..ff20e7e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,5 @@
 v1.8.1
+fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid)
 install: fix : correct man page directory (#387), reported by Stuart Cardall (@itoffshore)
 build  : `make` and `make test` compatible with `-jX`, reported by @mwgamera
 
diff --git a/lib/lz4.c b/lib/lz4.c
index f361b22..0960c97 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1127,7 +1127,6 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     BYTE* const oend = op + outputSize;
     BYTE* cpy;
     BYTE* oexit = op + targetOutputSize;
-    const BYTE* const lowLimit = lowPrefix - dictSize;
 
     const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
     const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};
@@ -1183,7 +1182,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* get offset */
         offset = LZ4_readLE16(ip); ip+=2;
         match = op - offset;
-        if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error;   /* Error : offset outside buffers */
+        if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error;   /* Error : offset outside buffers */
         LZ4_write32(op, (U32)offset);   /* costs ~1%; silence an msan warning when offset==0 */
 
         /* get matchlength */
-- 
cgit v0.12


From a2b4f732f4ab941dbd3108fb3c36c3afc54c47ee Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 10 Sep 2017 14:23:18 -0700
Subject: added -Wcomma to travisCI clang test

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2724e92..00cd40e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,7 +32,8 @@ matrix:
             - libc6-dev-i386
             - gcc-multilib
 
-    - env: Ubu=14.04 Cmd='make usan' COMPILER=clang
+    # presume clang >= v3.9.0
+    - env: Ubu=14.04 Cmd='make usan MOREFLAGS=-Wcomma' COMPILER=clang
       dist: trusty
       sudo: required
       addons:
-- 
cgit v0.12


From a30cba08f489c07e0d6085248287cd5dcc901b30 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 10 Sep 2017 14:32:38 -0700
Subject: fixed a bunch of -Wcomma warnings

reported by @rvandermeulen (#398)
---
 lib/lz4.c          | 6 +++++-
 programs/bench.c   | 5 ++++-
 programs/datagen.c | 5 ++++-
 programs/lz4cli.c  | 7 +++++--
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 0960c97..707b94c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -616,7 +616,11 @@ _next_match:
                 *token += ML_MASK;
                 matchCode -= ML_MASK;
                 LZ4_write32(op, 0xFFFFFFFF);
-                while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255;
+                while (matchCode >= 4*255) {
+                    op+=4;
+                    LZ4_write32(op, 0xFFFFFFFF);
+                    matchCode -= 4*255;
+                }
                 op += matchCode / 255;
                 *op++ = (BYTE)(matchCode % 255);
             } else
diff --git a/programs/bench.c b/programs/bench.c
index 05ddaff..5c83d59 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -428,7 +428,10 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
         DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
-        if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n;   /* buffer too small - stop after this file */
+        if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
+            fileSize = bufferSize-pos;
+            nbFiles=n;
+        }
         { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
           if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
           pos += readSize; }
diff --git a/programs/datagen.c b/programs/datagen.c
index a61afc0..7285d69 100644
--- a/programs/datagen.c
+++ b/programs/datagen.c
@@ -119,7 +119,10 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
     }
 
     /* init */
-    if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
+    if (pos==0) {
+        buffPtr[0] = RDG_genChar(seed, lt);
+        pos=1;
+    }
 
     /* Generate compressible data */
     while (pos < buffSize)
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index b4a3c14..ff489c6 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -259,8 +259,11 @@ static int exeNameMatch(const char* exeName, const char* test)
 static unsigned readU32FromChar(const char** stringPtr)
 {
     unsigned result = 0;
-    while ((**stringPtr >='0') && (**stringPtr <='9'))
-        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    while ((**stringPtr >='0') && (**stringPtr <='9')) {
+        result *= 10;
+        result += **stringPtr - '0';
+        (*stringPtr)++ ;
+    }
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         result <<= 10;
         if (**stringPtr=='M') result <<= 10;
-- 
cgit v0.12


From e5bd29a2fb019ddd19f9817ba701733552afbdbd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 10 Sep 2017 14:35:16 -0700
Subject: made clang warnings fail (-Werror)

in order to catch them in CI tests
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 00cd40e..3353dee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,7 +33,7 @@ matrix:
             - gcc-multilib
 
     # presume clang >= v3.9.0
-    - env: Ubu=14.04 Cmd='make usan MOREFLAGS=-Wcomma' COMPILER=clang
+    - env: Ubu=14.04 Cmd='make usan MOREFLAGS=-Wcomma -Werror' COMPILER=clang
       dist: trusty
       sudo: required
       addons:
-- 
cgit v0.12


From 810e2ca27b3561e0f6bfa7a88e0fde6faf807064 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 11 Sep 2017 10:25:47 -0700
Subject: minor improvements to examples

cosmetic : better display
added optional variable MOREFLAGS
---
 examples/Makefile                    | 20 ++++++++++++++------
 examples/blockStreaming_ringBuffer.c |  8 ++++----
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 7647f16..9321c24 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -30,11 +30,11 @@
 CPPFLAGS += -I../lib
 CFLAGS   ?= -O3
 CFLAGS   += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
-FLAGS    := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+FLAGS    := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
-TESTFILE= Makefile
-LZ4DIR := ../lib
-LZ4 = ../programs/lz4
+TESTFILE  = Makefile
+LZ4DIR   := ../lib
+LZ4       = ../programs/lz4
 
 
 # Define *.exe as extension for Windows systems
@@ -80,14 +80,22 @@ simpleBuffer: $(LZ4DIR)/lz4.c simple_buffer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 test : all
+	@echo "\n=== Print Version ==="
 	./printVersion$(EXT)
+	@echo "\n=== Simple compression example ==="
+	./simpleBuffer$(EXT)
+	@echo "\n=== Double-buffer ==="
 	./doubleBuffer$(EXT) $(TESTFILE)
-	./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
+	@echo "\n=== Ring Buffer ==="
 	./ringBuffer$(EXT)   $(TESTFILE)
+	@echo "\n=== Ring Buffer + LZ4 HC ==="
 	./ringBufferHC$(EXT) $(TESTFILE)
+	@echo "\n=== Compress line by line ==="
 	./lineCompress$(EXT) $(TESTFILE)
+	@echo "\n=== Dictionary Random Access ==="
+	./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
+	@echo "\n=== Frame compression ==="
 	./frameCompress$(EXT) $(TESTFILE)
-	./simpleBuffer$(EXT)
 	$(LZ4) -vt $(TESTFILE).lz4
 
 clean:
diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c
index 40109b5..dec78af 100644
--- a/examples/blockStreaming_ringBuffer.c
+++ b/examples/blockStreaming_ringBuffer.c
@@ -6,7 +6,7 @@
  * Compiler Options
  **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS // for MSVC
+#  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
 
@@ -24,7 +24,7 @@
 enum {
     MESSAGE_MAX_BYTES   = 1024,
     RING_BUFFER_BYTES   = 1024 * 8 + MESSAGE_MAX_BYTES,
-    DECODE_RING_BUFFER  = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES   // Intentionally larger, to test unsynchronized ring buffers
+    DECODE_RING_BUFFER  = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES   /* Intentionally larger, to test unsynchronized ring buffers */
 };
 
 
@@ -138,7 +138,7 @@ int main(int argc, char** argv)
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
-    if(argc < 2) {
+    if (argc < 2) {
         printf("Please specify input filename\n");
         return 0;
     }
@@ -176,7 +176,7 @@ int main(int argc, char** argv)
         FILE* const decFp = fopen(decFilename, "rb");
 
         const int cmp = compare(inpFp, decFp);
-        if(0 == cmp) {
+        if (0 == cmp) {
             printf("Verify : OK\n");
         } else {
             printf("Verify : NG\n");
-- 
cgit v0.12


From 77f7245ea7516d53f4ba5ecc7ee68899642c9736 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 22 Sep 2017 11:52:29 -0700
Subject: Fix Segfault When Copying Dict

dctx must have been initialized before we can copy the dictionary in.
---
 lib/lz4frame.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 3408708..166d583 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1616,6 +1616,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
     if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
       && (dctx->dict != dctx->tmpOutBuffer)
       && (dctx->dStage != dstage_getFrameHeader)
+      && (dctx->dStage != dstage_storeFrameHeader)
       && (!decompressOptionsPtr->stableDst)
       && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
     {
-- 
cgit v0.12


From ceb868f4425a547bc5e08b6c80aaf4cc750fc9aa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 23 Sep 2017 15:06:24 -0700
Subject: minor lz4frame code refactor

try to improve code readability.
minor optimization on condition to preserve history.
---
 lib/lz4frame.c | 104 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 54 insertions(+), 50 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 166d583..4df404c 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1253,29 +1253,33 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
                 dctx->tmpInSize += sizeToCopy;
                 srcPtr += sizeToCopy;
-                if (dctx->tmpInSize < dctx->tmpInTarget) {
-                    nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;   /* rest of header + nextBlockHeader */
-                    doAnotherStage = 0;   /* not enough src data, ask for some more */
-                    break;
-                }
-                {   size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget);  /* will update dStage appropriately */
-                    if (LZ4F_isError(hSize)) return hSize;
-                }
+            }
+            if (dctx->tmpInSize < dctx->tmpInTarget) {
+                nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;   /* rest of header + nextBlockHeader */
+                doAnotherStage = 0;   /* not enough src data, ask for some more */
                 break;
             }
+            {   size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget);  /* will update dStage appropriately */
+                if (LZ4F_isError(hSize)) return hSize;
+            }
+            break;
 
         case dstage_init:
             if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
             /* internal buffers allocation */
-            {   size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */;
+            {   size_t const bufferNeeded = dctx->maxBlockSize
+                    + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB)
+                    + 4 /* block checksum */;
                 if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
                     dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
                     FREEMEM(dctx->tmpIn);
                     dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
-                    if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    if (dctx->tmpIn == NULL)
+                        return err0r(LZ4F_ERROR_allocation_failed);
                     FREEMEM(dctx->tmpOutBuffer);
                     dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
-                    if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
+                    if (dctx->tmpOutBuffer== NULL)
+                        return err0r(LZ4F_ERROR_allocation_failed);
                     dctx->maxBufferSize = bufferNeeded;
             }   }
             dctx->tmpInSize = 0;
@@ -1299,18 +1303,20 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
             if (dctx->dStage == dstage_storeBlockHeader)   /* can be skipped */
         case dstage_storeBlockHeader:
-            {   size_t sizeToCopy = BHSize - dctx->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
+            {   size_t const remainingInput = (size_t)(srcEnd - srcPtr);
+                size_t const wantedData = BHSize - dctx->tmpInSize;
+                size_t const sizeToCopy = MIN(wantedData, remainingInput);
                 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
                 dctx->tmpInSize += sizeToCopy;
+
                 if (dctx->tmpInSize < BHSize) {   /* not enough input for cBlockSize */
                     nextSrcSizeHint = BHSize - dctx->tmpInSize;
                     doAnotherStage  = 0;
                     break;
                 }
                 selectedIn = dctx->tmpIn;
-            }
+            }   /* if (dctx->dStage == dstage_storeBlockHeader) */
 
         /* decode block header */
             {   size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
@@ -1414,7 +1420,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             break;
 
         case dstage_storeCBlock:
-            {   size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr));
+            {   size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;
+                size_t const inputLeft = (size_t)(srcEnd-srcPtr);
+                size_t const sizeToCopy = MIN(wantedData, inputLeft);
                 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
                 dctx->tmpInSize += sizeToCopy;
                 srcPtr += sizeToCopy;
@@ -1465,7 +1473,6 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
 
         case dstage_decodeCBlock_intoTmp:
             /* not enough place into dst : decode into tmpOut */
-
             /* ensure enough place for tmpOut */
             if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
                 if (dctx->dict == dctx->tmpOutBuffer) {
@@ -1518,29 +1525,27 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             }
 
         case dstage_getSuffix:
-            {   size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4;
-                if (dctx->frameRemainingSize)
-                    return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
-                if (suffixSize == 0) {  /* frame completed */
-                    nextSrcSizeHint = 0;
-                    LZ4F_resetDecompressionContext(dctx);
-                    doAnotherStage = 0;
-                    break;
-                }
-                if ((srcEnd - srcPtr) < 4) {  /* not enough size for entire CRC */
-                    dctx->tmpInSize = 0;
-                    dctx->dStage = dstage_storeSuffix;
-                } else {
-                    selectedIn = srcPtr;
-                    srcPtr += 4;
-                }
+            if (dctx->frameRemainingSize)
+                return err0r(LZ4F_ERROR_frameSize_wrong);   /* incorrect frame size decoded */
+            if (!dctx->frameInfo.contentChecksumFlag) {  /* no checksum, frame is completed */
+                nextSrcSizeHint = 0;
+                LZ4F_resetDecompressionContext(dctx);
+                doAnotherStage = 0;
+                break;
+            }
+            if ((srcEnd - srcPtr) < 4) {  /* not enough size for entire CRC */
+                dctx->tmpInSize = 0;
+                dctx->dStage = dstage_storeSuffix;
+            } else {
+                selectedIn = srcPtr;
+                srcPtr += 4;
             }
 
             if (dctx->dStage == dstage_storeSuffix)   /* can be skipped */
         case dstage_storeSuffix:
-            {
-                size_t sizeToCopy = 4 - dctx->tmpInSize;
-                if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
+            {   size_t const remainingInput = (size_t)(srcEnd - srcPtr);
+                size_t const wantedData = 4 - dctx->tmpInSize;
+                size_t const sizeToCopy = MIN(wantedData, remainingInput);
                 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
                 srcPtr += sizeToCopy;
                 dctx->tmpInSize += sizeToCopy;
@@ -1550,7 +1555,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     break;
                 }
                 selectedIn = dctx->tmpIn;
-            }
+            }   /* if (dctx->dStage == dstage_storeSuffix) */
 
         /* case dstage_checkSuffix: */   /* no direct call, avoid scan-build warning */
             {   U32 const readCRC = LZ4F_readLE32(selectedIn);
@@ -1589,7 +1594,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                     break;
                 }
                 selectedIn = dctx->header + 4;
-            }
+            }   /* if (dctx->dStage == dstage_storeSFrameSize) */
 
         /* case dstage_decodeSFrameSize: */   /* no direct access */
             {   size_t const SFrameSize = LZ4F_readLE32(selectedIn);
@@ -1606,22 +1611,22 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
                 doAnotherStage = 0;
                 nextSrcSizeHint = dctx->tmpInTarget;
                 if (nextSrcSizeHint) break;  /* still more to skip */
+                /* frame fully skipped : prepare context for a new frame */
                 LZ4F_resetDecompressionContext(dctx);
                 break;
             }
         }
-    }
-
-    /* preserve history within tmp if necessary */
-    if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
-      && (dctx->dict != dctx->tmpOutBuffer)
-      && (dctx->dStage != dstage_getFrameHeader)
-      && (dctx->dStage != dstage_storeFrameHeader)
-      && (!decompressOptionsPtr->stableDst)
-      && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
+    }   /* while (doAnotherStage) */
+
+    /* preserve history within tmp whenever necessary */
+    LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
+    if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)  /* next block will use up to 64KB from previous ones */
+      && (dctx->dict != dctx->tmpOutBuffer)             /* dictionary is not already within tmp */
+      && (!decompressOptionsPtr->stableDst)             /* cannot rely on dst data to remain there for next call */
+      && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) )  /* valid stages : [init ... getSuffix[ */
     {
         if (dctx->dStage == dstage_flushOut) {
-            size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
+            size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
             size_t copySize = 64 KB - dctx->tmpOutSize;
             const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
             if (dctx->tmpOutSize > 64 KB) copySize = 0;
@@ -1632,9 +1637,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             dctx->dict = dctx->tmpOutBuffer;
             dctx->dictSize = preserveSize + dctx->tmpOutStart;
         } else {
-            size_t newDictSize = dctx->dictSize;
-            const BYTE* oldDictEnd = dctx->dict + dctx->dictSize;
-            if ((newDictSize) > 64 KB) newDictSize = 64 KB;
+            const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;
+            size_t const newDictSize = MIN(dctx->dictSize, 64 KB);
 
             memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
 
-- 
cgit v0.12


From f6b31bf0d082315103d2fb5fc8f2094be567bb63 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 30 Sep 2017 10:35:55 -0700
Subject: fix #404

static analyzer `cppcheck` complains about a shift-by-32 on a 32 bits value,
which is an undefined behavior.
However, the flagged code path is never triggered in 32-bits mode,
(actually, it's not even generated if DCE kicks in),
the shift-by-32 is necessarily performed on a 64-bits value.

While it doesn't change anything regarding lz4 code generation, for both 32 and 64 bits mode,
(can be checked by md5sum on the generated binary),
the shift has been rewritten in a way which should please this static analyzer,
since it now pretends to shift by 16 on 32-bits cpu (note : it doesn't matter since the code will not even be generated in this case).

Note : this is a blind fix, the new code has not been tested with cppcheck, because cppcheck only works on Windows.
Other static analyzer, such as scan-build, do not trigger this false positive.
---
 lib/lz4.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 707b94c..19967f2 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -334,7 +334,7 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
 #       endif
         }
     } else   /* Big Endian CPU */ {
-        if (sizeof(val)==8) {
+        if (sizeof(val)==8) {   /* 64-bits */
 #       if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
             unsigned long r = 0;
             _BitScanReverse64( &r, val );
@@ -342,8 +342,11 @@ static unsigned LZ4_NbCommonBytes (register reg_t val)
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
             return (__builtin_clzll((U64)val) >> 3);
 #       else
+            static const U32 by32 = sizeof(val)*4;  /* 32 on 64 bits (goal), 16 on 32 bits.
+                Just to avoid some static analyzer complaining about shift by 32 on 32-bits target.
+                Note that this code path is never triggered in 32-bits mode. */
             unsigned r;
-            if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
+            if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; }
             if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
             r += (!val);
             return r;
-- 
cgit v0.12


From 87968517f926b55c2af3cf39e9692b368c566103 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 4 Oct 2017 15:24:08 -0700
Subject: fixed decoding block checksum in lz4frame

---
 lib/lz4frame.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 4df404c..3adbdd9 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1268,12 +1268,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
             /* internal buffers allocation */
             {   size_t const bufferNeeded = dctx->maxBlockSize
-                    + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB)
-                    + 4 /* block checksum */;
+                    + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
                 if (bufferNeeded > dctx->maxBufferSize) {   /* tmp buffers too small */
                     dctx->maxBufferSize = 0;   /* ensure allocation will be re-attempted on next entry*/
                     FREEMEM(dctx->tmpIn);
-                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
+                    dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize + 4 /* block checksum */);
                     if (dctx->tmpIn == NULL)
                         return err0r(LZ4F_ERROR_allocation_failed);
                     FREEMEM(dctx->tmpOutBuffer);
@@ -1407,7 +1406,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             dctx->dStage = dstage_getBlockHeader;  /* new block */
             break;
 
-        case dstage_getCBlock:   /* entry from dstage_decodeCBlockSize */
+        case dstage_getCBlock:
             if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
                 dctx->tmpInSize = 0;
                 dctx->dStage = dstage_storeCBlock;
-- 
cgit v0.12


From f1fa91d6fc74576dcde98b6514cfc8cc57f6f50f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 8 Oct 2017 23:40:21 -0700
Subject: insertAndFindBestMatch defers to insertAndGetWiderMatch

---
 lib/lz4hc.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index d7f8d23..f4a0981 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -116,7 +116,7 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
     hc4->nextToUpdate = target;
 }
 
-
+#if 0
 LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                                const BYTE* const ip, const BYTE* const iLimit,
                                                const BYTE** matchpos,
@@ -163,7 +163,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc
 
     return (int)ml;
 }
-
+#endif
 
 LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     LZ4HC_CCtx_internal* hc4,
@@ -234,6 +234,16 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     return longest;
 }
 
+LZ4_FORCE_INLINE
+int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
+                                const BYTE* const ip, const BYTE* const iLimit,
+                                const BYTE** matchpos,
+                                const int maxNbAttempts)
+{
+    const BYTE* uselessPtr = ip;
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
+}
+
 
 typedef enum {
     noLimit = 0,
@@ -353,7 +363,7 @@ static int LZ4HC_compress_hashChain (
     /* Main Loop */
     while (ip < mflimit) {
         ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
-        if (!ml) { ip++; continue; }
+        if (ml
Date: Sun, 8 Oct 2017 23:55:42 -0700
Subject: improved search of rep-1 patterns

---
 lib/lz4hc.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 131 insertions(+), 27 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index f4a0981..e28d682 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -165,6 +165,54 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc
 }
 #endif
 
+/** LZ4HC_countBack() :
+ * @return : negative value, nb of common bytes before ip/match */
+static int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
+                           const BYTE* const iMin, const BYTE* const mMin)
+{
+    int back=0;
+    while ( (ip+back > iMin)
+         && (match+back > mMin)
+         && (ip[back-1] == match[back-1]))
+            back--;
+    return back;
+}
+
+static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, reg_t pattern)
+{
+    const BYTE* const iStart = ip;
+
+    while (likely(ip=iLow+4)) {
+        if (LZ4_read32(ip-4) != pattern) break;
+        ip -= 4;
+    }
+    while (likely(ip>iLow)) {
+        if (ip[-1] != (BYTE)pattern) break;
+        ip--;
+    }
+
+    return (unsigned)(iStart - ip);
+}
+
+typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e;
+
 LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     LZ4HC_CCtx_internal* hc4,
     const BYTE* const ip,
@@ -180,11 +228,13 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const base = hc4->base;
     const U32 dictLimit = hc4->dictLimit;
     const BYTE* const lowPrefixPtr = base + dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
-    int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
+    reg_t const pattern = LZ4_read_ARCH(ip);
     U32 matchIndex;
+    repeat_state_e repeat = rep_untested;
+    size_t srcPatternLength = 0;
 
 
     /* First Match */
@@ -195,27 +245,29 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
-            if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
-                if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
-                    int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-                    int back = 0;
-
-                    while ( (ip+back > iLowLimit)
-                         && (matchPtr+back > lowPrefixPtr)
-                         && (ip[back-1] == matchPtr[back-1])) {
-                            back--;
-                    }
-
-                    mlt -= back;
+            if (LZ4_read32(matchPtr) == (U32)pattern) {
+                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+#if 0
+                /* more generic but unfortunately slower ... */
+                int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
+#else
+                int back = 0;
+                while ( (ip+back > iLowLimit)
+                     && (matchPtr+back > lowPrefixPtr)
+                     && (ip[back-1] == matchPtr[back-1])) {
+                        back--;
+                }
+#endif
+                mlt -= back;
 
-                    if (mlt > longest) {
-                        longest = mlt;
-                        *matchpos = matchPtr+back;
-                        *startpos = ip+back;
-            }   }   }
-        } else {
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = matchPtr+back;
+                    *startpos = ip+back;
+            }   }
+        } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
-            if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
+            if (LZ4_read32(matchPtr) == (U32)pattern) {
                 int mlt;
                 int back=0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
@@ -223,13 +275,65 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
                 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
                     mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
-                while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
+                while ( (ip+back > iLowLimit)
+                     && (matchIndex+back > lowLimit)
+                     && (ip[back-1] == matchPtr[back-1]))
+                        back--;
                 mlt -= back;
-                if (mlt > longest) { longest = mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
-            }
-        }
-        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
-    }
+                if (mlt > longest) {
+                    longest = mlt;
+                    *matchpos = base + matchIndex + back;
+                    *startpos = ip + back;
+        }   }   }
+
+        {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
+            matchIndex -= nextOffset;
+            if (1 && (nextOffset==1)) {
+                /* may be a repeated pattern */
+                if (repeat == rep_untested) {
+                    if (LZ4_read32(ip+4) == (U32)pattern) {  /* should check ip limit */
+                        repeat = rep_confirmed;
+                        srcPatternLength = LZ4HC_countPattern(ip+8, iHighLimit, pattern) + 8;
+                    } else {
+                        repeat = rep_not;
+                }   }
+                if ( (repeat == rep_confirmed)   /* proven repeated pattern (1-2-4) */
+                  && (matchIndex >= dictLimit) ) {   /* same segment only */
+                    const BYTE* const matchPtr = base + matchIndex;
+                    if (LZ4_read_ARCH(matchPtr) == pattern) {  /* good candidate */
+                        size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
+                        const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
+                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, (U32)pattern);
+                        size_t const currentSegmentLength = backLength + forwardPatternLength;
+
+                        if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
+                          && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
+#if 1
+                            matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
+#else
+                            const BYTE* const matchCandidate = matchPtr + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, pattern might be followed by more match */
+                            int matchLength = (int)(LZ4_count(ip + srcPatternLength, matchCandidate + srcPatternLength, iHighLimit) + srcPatternLength);
+                            int back = 0;
+                            while ( (ip+back > iLowLimit)
+                                 && (matchPtr+back > lowPrefixPtr)
+                                 && (ip[back-1] == matchPtr[back-1])) {
+                                    back--;
+                            }
+                            matchLength -= back;
+                            if (matchLength > longest) {
+                                longest = matchLength;
+                                *matchpos = base + matchIndex + back;
+                                *startpos = ip + back;
+                            }
+                            matchIndex -= (U32)backLength;
+                            matchIndex -= DELTANEXTU16(chainTable, matchIndex);   /* skip directly to next potential pattern segment */
+#endif
+                        } else {
+                            matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
+                            //matchIndex -= DELTANEXTU16(chainTable, matchIndex);   /* skip directly to following candidate; slightly faster, but miss some rare corner cases (likely when back is useful)*/
+                        }
+        }   }   }   }
+    }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
 
     return longest;
 }
-- 
cgit v0.12


From 1ee17e4eb8268aecbec3611f77061a38478a22ad Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 9 Oct 2017 00:31:12 -0700
Subject: optional fuse

---
 lib/lz4hc.c | 121 ++++++++++++++++++++++++++----------------------------------
 1 file changed, 53 insertions(+), 68 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index e28d682..010ac2d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -116,55 +116,6 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
     hc4->nextToUpdate = target;
 }
 
-#if 0
-LZ4_FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
-                                               const BYTE* const ip, const BYTE* const iLimit,
-                                               const BYTE** matchpos,
-                                               const int maxNbAttempts)
-{
-    U16* const chainTable = hc4->chainTable;
-    U32* const HashTable = hc4->hashTable;
-    const BYTE* const base = hc4->base;
-    const BYTE* const dictBase = hc4->dictBase;
-    const U32 dictLimit = hc4->dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
-    U32 matchIndex;
-    int nbAttempts = maxNbAttempts;
-    size_t ml = 0;
-
-    /* HC4 match finder */
-    LZ4HC_Insert(hc4, ip);
-    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
-        nbAttempts--;
-        if (matchIndex >= dictLimit) {
-            const BYTE* const match = base + matchIndex;
-            if ( (*(match+ml) == *(ip+ml))   /* can be longer */
-               && (LZ4_read32(match) == LZ4_read32(ip)) )
-            {
-                size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
-                if (mlt > ml) { ml = mlt; *matchpos = match; }
-            }
-        } else {
-            const BYTE* const match = dictBase + matchIndex;
-            if (LZ4_read32(match) == LZ4_read32(ip)) {
-                size_t mlt;
-                const BYTE* vLimit = ip + (dictLimit - matchIndex);
-                if (vLimit > iLimit) vLimit = iLimit;
-                mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
-                if ((ip+mlt == vLimit) && (vLimit < iLimit))
-                    mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
-                if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
-            }
-        }
-        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
-    }
-
-    return (int)ml;
-}
-#endif
-
 /** LZ4HC_countBack() :
  * @return : negative value, nb of common bytes before ip/match */
 static int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
@@ -308,26 +259,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
-#if 1
                             matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
-#else
-                            const BYTE* const matchCandidate = matchPtr + (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, pattern might be followed by more match */
-                            int matchLength = (int)(LZ4_count(ip + srcPatternLength, matchCandidate + srcPatternLength, iHighLimit) + srcPatternLength);
-                            int back = 0;
-                            while ( (ip+back > iLowLimit)
-                                 && (matchPtr+back > lowPrefixPtr)
-                                 && (ip[back-1] == matchPtr[back-1])) {
-                                    back--;
-                            }
-                            matchLength -= back;
-                            if (matchLength > longest) {
-                                longest = matchLength;
-                                *matchpos = base + matchIndex + back;
-                                *startpos = ip + back;
-                            }
-                            matchIndex -= (U32)backLength;
-                            matchIndex -= DELTANEXTU16(chainTable, matchIndex);   /* skip directly to next potential pattern segment */
-#endif
                         } else {
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
                             //matchIndex -= DELTANEXTU16(chainTable, matchIndex);   /* skip directly to following candidate; slightly faster, but miss some rare corner cases (likely when back is useful)*/
@@ -338,6 +270,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     return longest;
 }
 
+#if 0
 LZ4_FORCE_INLINE
 int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                 const BYTE* const ip, const BYTE* const iLimit,
@@ -348,6 +281,58 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
 }
 
+#else
+
+LZ4_FORCE_INLINE
+int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
+                                const BYTE* const ip, const BYTE* const iLimit,
+                                const BYTE** matchpos,
+                                const int maxNbAttempts)
+{
+    U16* const chainTable = hc4->chainTable;
+    U32* const HashTable = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const BYTE* const dictBase = hc4->dictBase;
+    const U32 dictLimit = hc4->dictLimit;
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    U32 matchIndex;
+    int nbAttempts = maxNbAttempts;
+    size_t ml = 0;
+
+    /* HC4 match finder */
+    LZ4HC_Insert(hc4, ip);
+    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+
+    while ((matchIndex>=lowLimit) && (nbAttempts)) {
+        nbAttempts--;
+        if (matchIndex >= dictLimit) {
+            const BYTE* const match = base + matchIndex;
+            if ( (*(match+ml) == *(ip+ml))   /* can be longer */
+               && (LZ4_read32(match) == LZ4_read32(ip)) )
+            {
+                size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
+                if (mlt > ml) { ml = mlt; *matchpos = match; }
+            }
+        } else {
+            const BYTE* const match = dictBase + matchIndex;
+            if (LZ4_read32(match) == LZ4_read32(ip)) {
+                size_t mlt;
+                const BYTE* vLimit = ip + (dictLimit - matchIndex);
+                if (vLimit > iLimit) vLimit = iLimit;
+                mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
+                if ((ip+mlt == vLimit) && (vLimit < iLimit))
+                    mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
+                if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
+            }
+        }
+        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
+    }
+
+    return (int)ml;
+}
+#endif
+
+
 
 typedef enum {
     noLimit = 0,
-- 
cgit v0.12


From bdca63ed69dc1873b385c87b25d64fa5ebd595f7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 9 Oct 2017 00:36:47 -0700
Subject: early out is not better

---
 lib/README.md | 2 +-
 lib/lz4hc.c   | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/README.md b/lib/README.md
index 7082fe3..fc5d4e9 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -15,7 +15,7 @@ They generate and decode data using [LZ4 block format].
 
 For more compression ratio at the cost of compression speed,
 the High Compression variant called **lz4hc** is available.
-Add files **`lz4hc.c`** and **`lz4hc.h`**.
+Add files **`lz4hc.c`**, **`lz4hc.h`** and **`lz4opt.h`**.
 The variant still depends on regular `lib/lz4.*` source files.
 
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 010ac2d..19636c5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -262,7 +262,6 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                             matchIndex += (U32)forwardPatternLength - (U32)srcPatternLength;  /* best position, full pattern, might be followed by more match */
                         } else {
                             matchIndex -= (U32)backLength;   /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */
-                            //matchIndex -= DELTANEXTU16(chainTable, matchIndex);   /* skip directly to following candidate; slightly faster, but miss some rare corner cases (likely when back is useful)*/
                         }
         }   }   }   }
     }  /* while ((matchIndex>=lowLimit) && (nbAttempts)) */
-- 
cgit v0.12


From 97c18f5f0edf89fb7846bc2923bdc1568a2d95c1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 9 Oct 2017 01:44:05 -0700
Subject: re-inserted last byte test in widerMatch

---
 lib/lz4hc.c | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 19636c5..bd75911 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -181,6 +181,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const lowPrefixPtr = base + dictLimit;
     const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE;
     const BYTE* const dictBase = hc4->dictBase;
+    int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
     reg_t const pattern = LZ4_read_ARCH(ip);
     U32 matchIndex;
@@ -196,26 +197,28 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
-            if (LZ4_read32(matchPtr) == (U32)pattern) {
-                int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
-#if 0
-                /* more generic but unfortunately slower ... */
-                int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
-#else
-                int back = 0;
-                while ( (ip+back > iLowLimit)
-                     && (matchPtr+back > lowPrefixPtr)
-                     && (ip[back-1] == matchPtr[back-1])) {
-                        back--;
-                }
-#endif
-                mlt -= back;
+            if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
+                if (LZ4_read32(matchPtr) == (U32)pattern) {
+                    int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+    #if 0
+                    /* more generic but unfortunately slower ... */
+                    int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
+    #else
+                    int back = 0;
+                    while ( (ip+back > iLowLimit)
+                         && (matchPtr+back > lowPrefixPtr)
+                         && (ip[back-1] == matchPtr[back-1])) {
+                            back--;
+                    }
+    #endif
+                    mlt -= back;
 
-                if (mlt > longest) {
-                    longest = mlt;
-                    *matchpos = matchPtr+back;
-                    *startpos = ip+back;
-            }   }
+                    if (mlt > longest) {
+                        longest = mlt;
+                        *matchpos = matchPtr+back;
+                        *startpos = ip+back;
+                }   }
+            }
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
             if (LZ4_read32(matchPtr) == (U32)pattern) {
-- 
cgit v0.12


From a4314829db575453911046e2b0d5a19697b14a67 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 9 Oct 2017 01:50:28 -0700
Subject: fused getLongerMatch and getWiderMatch

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index bd75911..47474d3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -272,7 +272,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     return longest;
 }
 
-#if 0
+#if 1
 LZ4_FORCE_INLINE
 int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                 const BYTE* const ip, const BYTE* const iLimit,
-- 
cgit v0.12


From 2bd85f41994e9695911cfc4c86fbc04fdb35ee82 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 22 Sep 2017 11:55:42 -0700
Subject: Add Dictionary Support to the Command Line Tool

---
 programs/lz4cli.c | 27 ++++++++++++++++++
 programs/lz4io.c  | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 programs/lz4io.h  |  2 ++
 3 files changed, 109 insertions(+), 4 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index ff489c6..857fa65 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -113,6 +113,7 @@ static int usage(const char* exeName)
     DISPLAY( " -9     : High compression \n");
     DISPLAY( " -d     : decompression (default for %s extension)\n", LZ4_EXTENSION);
     DISPLAY( " -z     : force compression \n");
+    DISPLAY( " -D FILE: use dictionary in FILE \n");
     DISPLAY( " -f     : overwrite output without prompting \n");
     DISPLAY( " -k     : preserve source files(s)  (default) \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
@@ -290,6 +291,7 @@ int main(int argc, const char** argv)
     operationMode_e mode = om_auto;
     const char* input_filename = NULL;
     const char* output_filename= NULL;
+    const char* dictionary_filename = NULL;
     char* dynNameSpace = NULL;
     const char** inFileNames = (const char**) calloc(argc, sizeof(char*));
     unsigned ifnIdx=0;
@@ -399,6 +401,22 @@ int main(int argc, const char** argv)
                     /* Compression (default) */
                 case 'z': mode = om_compress; break;
 
+                case 'D':
+                    if (argument[1] == '\0') {
+                        /* path is next arg */
+                        if (i + 1 == argc) {
+                            /* there is no next arg */
+                            badusage(exeName);
+                        }
+                        dictionary_filename = argv[++i];
+                    } else {
+                        /* path follows immediately */
+                        dictionary_filename = argument + 1;
+                    }
+                    /* skip to end of argument so that we jump to parsing next argument */
+                    argument += strlen(argument) - 1;
+                    break;
+
                     /* Use Legacy format (ex : Linux kernel compression) */
                 case 'l': legacy_format = 1; blockSize = 8 MB; break;
 
@@ -560,6 +578,15 @@ int main(int argc, const char** argv)
         mode = om_decompress;   /* defer to decompress */
     }
 
+    if (dictionary_filename) {
+        if (!strcmp(dictionary_filename, stdinmark) && IS_CONSOLE(stdin)) {
+            DISPLAYLEVEL(1, "refusing to read from a console\n");
+            exit(1);
+        }
+
+        LZ4IO_setDictionaryFilename(dictionary_filename);
+    }
+
     /* compress or decompress */
     if (!input_filename) input_filename = stdinmark;
     /* Check if input is defined as console; trigger an error in this case */
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 06741b4..642e11c 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -57,6 +57,7 @@
 #include "lz4.h"       /* still required for legacy format */
 #include "lz4hc.h"     /* still required for legacy format */
 #include "lz4frame.h"
+#include "lz4frame_static.h"
 
 
 /*****************************
@@ -110,6 +111,8 @@ static int g_streamChecksum = 1;
 static int g_blockIndependence = 1;
 static int g_sparseFileSupport = 1;
 static int g_contentSizeFlag = 0;
+static int g_useDictionary = 0;
+static const char* g_dictionaryFilename = NULL;
 
 
 /**************************************
@@ -142,6 +145,12 @@ static int g_contentSizeFlag = 0;
 /* ****************** Parameters ******************** */
 /* ************************************************** */
 
+int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) {
+    g_dictionaryFilename = dictionaryFilename;
+    g_useDictionary = dictionaryFilename != NULL;
+    return g_useDictionary;
+}
+
 /* Default setting : overwrite = 1; return : overwrite mode (0/1) */
 int LZ4IO_setOverwrite(int yes)
 {
@@ -395,8 +404,53 @@ typedef struct {
     void*  dstBuffer;
     size_t dstBufferSize;
     LZ4F_compressionContext_t ctx;
+    LZ4F_CDict* cdict;
 } cRess_t;
 
+static void* LZ4IO_createDict(const char* dictionaryFilename, size_t *dictionarySize) {
+    FILE* dictionaryFile;
+    size_t blockSize = 64 KB;
+    size_t dictionaryBufferSize = blockSize;
+    size_t readSize;
+    void* dictionaryBuffer;
+    *dictionarySize = 0;
+    dictionaryBuffer = malloc(dictionaryBufferSize);
+
+    if (!dictionaryBuffer) EXM_THROW(25, "Allocation error : not enough memory");
+
+    if (!dictionaryFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+
+    dictionaryFile = LZ4IO_openSrcFile(g_dictionaryFilename);
+    if (!dictionaryFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+
+    do {
+        if (*dictionarySize + blockSize > dictionaryBufferSize) {
+            dictionaryBufferSize *= 2;
+            dictionaryBuffer = realloc(dictionaryBuffer, dictionaryBufferSize);
+            if (!dictionaryBuffer) EXM_THROW(26, "Allocation error : not enough memory");
+        }
+        /* Read next block */
+        readSize = fread((char*)dictionaryBuffer + *dictionarySize, (size_t)1, (size_t)blockSize, dictionaryFile);
+        *dictionarySize += readSize;
+    } while (readSize>0);
+
+    return dictionaryBuffer;
+}
+
+static LZ4F_CDict* LZ4IO_createCDict(void) {
+    size_t dictionarySize;
+    void* dictionaryBuffer;
+    LZ4F_CDict* cdict;
+    if (!g_useDictionary) {
+        return NULL;
+    }
+    dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize);
+    if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+    cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
+    free(dictionaryBuffer);
+    return cdict;
+}
+
 static cRess_t LZ4IO_createCResources(void)
 {
     const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
@@ -412,6 +466,8 @@ static cRess_t LZ4IO_createCResources(void)
     ress.dstBuffer = malloc(ress.dstBufferSize);
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
 
+    ress.cdict = LZ4IO_createCDict();
+
     return ress;
 }
 
@@ -419,6 +475,10 @@ static void LZ4IO_freeCResources(cRess_t ress)
 {
     free(ress.srcBuffer);
     free(ress.dstBuffer);
+
+    LZ4F_freeCDict(ress.cdict);
+    ress.cdict = NULL;
+
     { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
       if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
 }
@@ -472,7 +532,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     /* single-block file */
     if (readSize < blockSize) {
         /* Compress in single pass */
-        size_t const cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs);
+        size_t cSize = LZ4F_compressFrame_usingCDict(dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
         if (LZ4F_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
         compressedfilesize = cSize;
         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
@@ -488,7 +548,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
     /* multiple-blocks file */
     {
         /* Write Archive Header */
-        size_t headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs);
+        size_t headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
         if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
         { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile);
           if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); }
@@ -745,8 +805,21 @@ typedef struct {
     size_t dstBufferSize;
     FILE*  dstFile;
     LZ4F_decompressionContext_t dCtx;
+    void*  dictBuffer;
+    size_t dictBufferSize;
 } dRess_t;
 
+static void LZ4IO_loadDDict(dRess_t* ress) {
+    if (!g_useDictionary) {
+        ress->dictBuffer = NULL;
+        ress->dictBufferSize = 0;
+        return;
+    }
+
+    ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize);
+    if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
+}
+
 static const size_t LZ4IO_dBufferSize = 64 KB;
 static dRess_t LZ4IO_createDResources(void)
 {
@@ -763,6 +836,8 @@ static dRess_t LZ4IO_createDResources(void)
     ress.dstBuffer = malloc(ress.dstBufferSize);
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
 
+    LZ4IO_loadDDict(&ress);
+
     ress.dstFile = NULL;
     return ress;
 }
@@ -773,6 +848,7 @@ static void LZ4IO_freeDResources(dRess_t ress)
     if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
     free(ress.srcBuffer);
     free(ress.dstBuffer);
+    free(ress.dictBuffer);
 }
 
 
@@ -786,7 +862,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
     {   size_t inSize = MAGICNUMBER_SIZE;
         size_t outSize= 0;
         LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
-        nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
+        nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
         if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
     }
 
@@ -805,7 +881,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
             /* Decode Input (at least partially) */
             size_t remaining = readSize - pos;
             decodedBytes = ress.dstBufferSize;
-            nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
+            nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
             if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
             pos += remaining;
 
diff --git a/programs/lz4io.h b/programs/lz4io.h
index 6190f00..b21b8b6 100644
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -64,6 +64,8 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz
 /* ****************** Parameters ******************** */
 /* ************************************************** */
 
+int LZ4IO_setDictionaryFilename(const char* dictionaryFilename);
+
 /* Default setting : overwrite = 1;
    return : overwrite mode (0/1) */
 int LZ4IO_setOverwrite(int yes);
-- 
cgit v0.12


From 93f8284c175a4047b0d9df3112927bbb3b832b2a Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Fri, 22 Sep 2017 14:50:11 -0700
Subject: Add some tests verifying command line dictionary functionality

---
 tests/Makefile | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index e870fcf..1a907b7 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -129,6 +129,8 @@ ifneq (,$(filter $(shell uname),SunOS))
 DIFF:=gdiff
 endif
 
+DD:=dd
+
 
 test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer
 
@@ -253,6 +255,31 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
 	$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv  # test block checksum
 	@$(RM) tmp-tlb*
 
+test-lz4-dict: lz4 datagen
+	@echo "\n ---- test lz4 compression/decompression with dictionary ----"
+	./datagen -g16KB > tmp-dict
+	./datagen -g32KB > tmp-dict-sample-32k
+	< tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
+	./datagen -g128MB > tmp-dict-sample-128m
+	< tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
+	touch tmp-dict-sample-0
+	< tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
+
+	< tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k
+	< tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
+
+	@echo "\n ---- test lz4 dictionary loading ----"
+	./datagen -g128KB > tmp-dict-data-128KB
+	set -e; \
+	for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
+		./datagen -g$$l > tmp-dict-$$l; \
+		$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
+		< tmp-dict-$$l      $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
+		< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB | $(LZ4) -dD tmp-dict-$$l      | $(DIFF) - tmp-dict-data-128KB; \
+	done
+
+	@$(RM) tmp-dict*
+
 test-lz4-hugefile: lz4 datagen
 	@echo "\n ---- test huge files compression/decompression ----"
 	./datagen -g6GB   | $(LZ4) -vB5D  | $(LZ4) -qt
@@ -292,7 +319,7 @@ test-lz4-opt-parser: lz4 datagen
 
 test-lz4: lz4 datagen test-lz4-basic test-lz4-opt-parser test-lz4-multiple \
           test-lz4-sparse test-lz4-frame-concatenation test-lz4-testmode \
-          test-lz4-contentSize test-lz4-hugefile
+          test-lz4-contentSize test-lz4-hugefile test-lz4-dict
 	@$(RM) tmp*
 
 test-lz4c: lz4c datagen
-- 
cgit v0.12


From 9a16272261f571d54e4642c760d099adb6cc27b1 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Tue, 3 Oct 2017 12:50:28 -0400
Subject: Read the Dictionary into a Circular Buffer

---
 programs/lz4io.c | 71 ++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 49 insertions(+), 22 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 642e11c..57434f7 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -83,6 +83,7 @@
 #define LEGACY_BLOCKSIZE   (8 MB)
 #define MIN_STREAM_BUFSIZE (192 KB)
 #define LZ4IO_BLOCKSIZEID_DEFAULT 7
+#define LZ4_MAX_DICT_SIZE (64 KB)
 
 
 /**************************************
@@ -407,34 +408,60 @@ typedef struct {
     LZ4F_CDict* cdict;
 } cRess_t;
 
-static void* LZ4IO_createDict(const char* dictionaryFilename, size_t *dictionarySize) {
-    FILE* dictionaryFile;
-    size_t blockSize = 64 KB;
-    size_t dictionaryBufferSize = blockSize;
+static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) {
     size_t readSize;
-    void* dictionaryBuffer;
-    *dictionarySize = 0;
-    dictionaryBuffer = malloc(dictionaryBufferSize);
+    size_t dictEnd = 0;
+    size_t dictLen = 0;
+    size_t dictStart;
+    size_t circularBufSize = LZ4_MAX_DICT_SIZE;
+    char* circularBuf;
+    char* dictBuf;
+    FILE* dictFile;
+
+    if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+
+    circularBuf = (char *) malloc(circularBufSize);
+    if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory");
+
+    dictFile = LZ4IO_openSrcFile(dictFilename);
+    if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+
+    /* opportunistically seek to the part of the file we care about. If this */
+    /* fails it's not a problem since we'll just read everything anyways.    */
+    if (strcmp(dictFilename, stdinmark)) {
+        UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
+    }
 
-    if (!dictionaryBuffer) EXM_THROW(25, "Allocation error : not enough memory");
+    do {
+        readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile);
+        dictEnd = (dictEnd + readSize) % circularBufSize;
+        dictLen += readSize;
+    } while (readSize>0);
 
-    if (!dictionaryFilename) EXM_THROW(25, "Dictionary error : no filename provided");
+    if (dictLen > LZ4_MAX_DICT_SIZE) {
+        dictLen = LZ4_MAX_DICT_SIZE;
+    }
 
-    dictionaryFile = LZ4IO_openSrcFile(g_dictionaryFilename);
-    if (!dictionaryFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
+    *dictSize = dictLen;
 
-    do {
-        if (*dictionarySize + blockSize > dictionaryBufferSize) {
-            dictionaryBufferSize *= 2;
-            dictionaryBuffer = realloc(dictionaryBuffer, dictionaryBufferSize);
-            if (!dictionaryBuffer) EXM_THROW(26, "Allocation error : not enough memory");
-        }
-        /* Read next block */
-        readSize = fread((char*)dictionaryBuffer + *dictionarySize, (size_t)1, (size_t)blockSize, dictionaryFile);
-        *dictionarySize += readSize;
-    } while (readSize>0);
+    dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize;
+
+    if (dictStart == 0) {
+        /* We're in the simple case where the dict starts at the beginning of our circular buffer. */
+        dictBuf = circularBuf;
+        circularBuf = NULL;
+    } else {
+        /* Otherwise, we will alloc a new buffer and copy our dict into that. */
+        dictBuf = (char *) malloc(dictLen ? dictLen : 1);
+        if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
+
+        memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
+        memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
+    }
+
+    free(circularBuf);
 
-    return dictionaryBuffer;
+    return dictBuf;
 }
 
 static LZ4F_CDict* LZ4IO_createCDict(void) {
-- 
cgit v0.12


From 73bcf90e51b9837454b92265d30d4a214be38b02 Mon Sep 17 00:00:00 2001
From: Rei Odaira 
Date: Fri, 13 Oct 2017 14:53:37 -0500
Subject: Use the optimization level of O2 for the decompression functions on
 ppc64le with gcc, to avoid harmful unrolling and SIMDization with O3

---
 lib/lz4.c | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 19967f2..179408d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -117,6 +117,28 @@
 #  endif  /* _MSC_VER */
 #endif /* LZ4_FORCE_INLINE */
 
+/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE
+ * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy,
+ * together with a simple 8-byte copy loop as a fall-back path.
+ * However, this optimization hurts the decompression speed by >30%,
+ * because the execution does not go to the optimized loop
+ * for typical compressible data, and all of the preamble checks
+ * before going to the fall-back path become useless overhead.
+ * This optimization happens only with the -O3 flag, and -O2 generates
+ * a simple 8-byte copy loop.
+ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy
+ * functions are annotated with __attribute__((optimize("O2"))),
+ * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute
+ * of LZ4_wildCopy does not affect the compression speed.
+ */
+#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__)
+#  define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2")))
+#  define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE
+#else
+#  define LZ4_FORCE_O2_GCC_PPC64LE
+#  define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static
+#endif
+
 #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
 #  define expect(expr,value)    (__builtin_expect ((expr),(value)) )
 #else
@@ -253,7 +275,8 @@ static void LZ4_copy8(void* dst, const void* src)
 }
 
 /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
-static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
+LZ4_FORCE_O2_INLINE_GCC_PPC64LE
+void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
 {
     BYTE* d = (BYTE*)dstPtr;
     const BYTE* s = (const BYTE*)srcPtr;
@@ -1112,6 +1135,7 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
  *  Note that it is important for performance that this function really get inlined,
  *  in order to remove useless branches during compilation optimization.
  */
+LZ4_FORCE_O2_GCC_PPC64LE
 LZ4_FORCE_INLINE int LZ4_decompress_generic(
                  const char* const src,
                  char* const dst,
@@ -1272,16 +1296,19 @@ _output_error:
 }
 
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
     return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB);
@@ -1327,6 +1354,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti
     If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setStreamDecode()
 */
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
@@ -1353,6 +1381,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
     return result;
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
 {
     LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse;
@@ -1387,6 +1416,7 @@ Advanced decoding functions :
     the dictionary must be explicitly provided within parameters
 */
 
+LZ4_FORCE_O2_GCC_PPC64LE
 LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
 {
     if (dictSize==0)
@@ -1399,17 +1429,20 @@ LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char*
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
 {
     return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
 }
 
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
 {
     return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
 }
 
 /* debug function */
+LZ4_FORCE_O2_GCC_PPC64LE
 int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
 {
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
-- 
cgit v0.12


From 2d827e1b287881fbdef00b9d88060d63e8a66236 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 14 Oct 2017 18:48:00 -0700
Subject: lz4cli : removed extension artefacts

It used to be useful for an old Windows variant which is no longer maintained.
---
 programs/lz4cli.c | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 857fa65..05cb69f 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -90,9 +90,6 @@ static unsigned displayLevel = 2;   /* 0 : no display ; 1: errors only ; 2 : dow
 /*-************************************
 *  Version modifiers
 ***************************************/
-#define EXTENDED_ARGUMENTS
-#define EXTENDED_HELP
-#define EXTENDED_FORMAT
 #define DEFAULT_COMPRESSOR   LZ4IO_compressFilename
 #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
 int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel);   /* hidden function */
@@ -156,7 +153,6 @@ static int usage_advanced(const char* exeName)
         DISPLAY( " -hc    : high compression \n");
         DISPLAY( " -y     : overwrite output without prompting \n");
     }
-    EXTENDED_HELP;
     return 0;
 }
 
@@ -506,9 +502,6 @@ int main(int argc, const char** argv)
                     /* Pause at the end (hidden option) */
                 case 'p': main_pause=1; break;
 
-                    /* Specific commands for customized versions */
-                EXTENDED_ARGUMENTS;
-
                     /* Unrecognised command */
                 default : badusage(exeName);
                 }
@@ -560,8 +553,7 @@ int main(int argc, const char** argv)
                 free((void*)inFileNames);
                 inFileNames = extendedFileList;
                 ifnIdx = fileNamesNb;
-            }
-        }
+        }   }
 #endif
     }
 
@@ -583,7 +575,6 @@ int main(int argc, const char** argv)
             DISPLAYLEVEL(1, "refusing to read from a console\n");
             exit(1);
         }
-
         LZ4IO_setDictionaryFilename(dictionary_filename);
     }
 
@@ -654,7 +645,7 @@ int main(int argc, const char** argv)
             operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename);
     } else {   /* compression is default action */
         if (legacy_format) {
-            DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n");
+            DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n");
             LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
         } else {
             if (multiple_inputs)
@@ -666,12 +657,13 @@ int main(int argc, const char** argv)
 
 _cleanup:
     if (main_pause) waitEnter();
-    if (dynNameSpace) free(dynNameSpace);
+    free(dynNameSpace);
 #ifdef UTIL_HAS_CREATEFILELIST
-    if (extendedFileList)
+    if (extendedFileList) {
         UTIL_freeFileList(extendedFileList, fileNamesBuf);
-    else
+        inFileNames = NULL;
+    }
 #endif
-        free((void*)inFileNames);
+    free((void*)inFileNames);
     return operationResult;
 }
-- 
cgit v0.12


From dccf8826f1d76efcbdc655e63cc04cdbd1123619 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 14 Oct 2017 23:50:07 -0700
Subject: lz4cli : minor rewrite of lz4c legacy commands

for clarity
---
 programs/lz4cli.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 05cb69f..362ba49 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -150,7 +150,7 @@ static int usage_advanced(const char* exeName)
         DISPLAY( "Legacy arguments : \n");
         DISPLAY( " -c0    : fast compression \n");
         DISPLAY( " -c1    : high compression \n");
-        DISPLAY( " -hc    : high compression \n");
+        DISPLAY( " -c2,-hc: very high compression \n");
         DISPLAY( " -y     : overwrite output without prompting \n");
     }
     return 0;
@@ -205,14 +205,14 @@ static int usage_longhelp(const char* exeName)
     DISPLAY( "          generator | %s | consumer \n", exeName);
     if (g_lz4c_legacy_commands) {
         DISPLAY( "\n");
-        DISPLAY( "***** Warning  *****\n");
+        DISPLAY( "***** Warning  ***** \n");
         DISPLAY( "Legacy arguments take precedence. Therefore : \n");
-        DISPLAY( "---------------------------------\n");
-        DISPLAY( "          %s -hc filename\n", exeName);
-        DISPLAY( "means 'compress filename in high compression mode'\n");
-        DISPLAY( "It is not equivalent to :\n");
-        DISPLAY( "          %s -h -c filename\n", exeName);
-        DISPLAY( "which would display help text and exit\n");
+        DISPLAY( "--------------------------------- \n");
+        DISPLAY( "          %s -hc filename \n", exeName);
+        DISPLAY( "means 'compress filename in high compression mode' \n");
+        DISPLAY( "It is not equivalent to : \n");
+        DISPLAY( "          %s -h -c filename \n", exeName);
+        DISPLAY( "which displays help text and exits \n");
     }
     return 0;
 }
@@ -368,10 +368,11 @@ int main(int argc, const char** argv)
 
                 if (g_lz4c_legacy_commands) {
                     /* Legacy commands (-c0, -c1, -hc, -y) */
-                    if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; }  /* -c0 (fast compression) */
-                    if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; }  /* -c1 (high compression) */
-                    if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; }  /* -hc (high compression) */
-                    if (argument[0]=='y') { LZ4IO_setOverwrite(1); continue; }                         /* -y (answer 'yes' to overwrite permission) */
+                    if (!strcmp(argument,  "c0")) { cLevel=0; argument++; continue; }  /* -c0 (fast compression) */
+                    if (!strcmp(argument,  "c1")) { cLevel=9; argument++; continue; }  /* -c1 (high compression) */
+                    if (!strcmp(argument,  "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */
+                    if (!strcmp(argument,  "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */
+                    if (!strcmp(argument,  "y"))  { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */
                 }
 
                 if ((*argument>='0') && (*argument<='9')) {
-- 
cgit v0.12


From 74d8688bc8104d897f9c98fc747400e71fbe77bc Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 17 Oct 2017 12:42:27 -0700
Subject: [bench] Use higher resolution timer on POSIX

The timer used was only accurate up to 0.01 seconds. This timer is accurate up to 1 ns.
It is a monotonic timer that measures the real time difference, not on CPU time.

Copied the benchmark code from https://github.com/facebook/zstd/commit/6ab4d5e9041aba962a810ffee191f95897c6208e
---
 programs/bench.c |  24 +++++------
 programs/util.h  | 127 +++++++++++++++++++++++++++++++++++++++++++------------
 2 files changed, 110 insertions(+), 41 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 5c83d59..fac2a87 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -166,7 +166,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
     U32 nbBlocks;
-    UTIL_time_t ticksPerSecond;
     struct compressionParameters compP;
     int cfunctionId;
 
@@ -176,7 +175,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
 
     /* init */
     if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
-    UTIL_initTimer(&ticksPerSecond);
 
     /* Init */
     if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1;
@@ -229,17 +227,17 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         size_t cSize = 0;
         double ratio = 0.;
 
-        UTIL_getTime(&coolTime);
+        coolTime = UTIL_getTime();
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted || !dCompleted) {
             UTIL_time_t clockStart;
             U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
 
             /* overheat protection */
-            if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
+            if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
                 DISPLAYLEVEL(2, "\rcooling down ...    \r");
                 UTIL_sleep(COOLPERIOD_SEC);
-                UTIL_getTime(&coolTime);
+                coolTime = UTIL_getTime();
             }
 
             /* Compression */
@@ -247,8 +245,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
 
             UTIL_sleepMilli(1);  /* give processor time to other processes */
-            UTIL_waitForNextTick(ticksPerSecond);
-            UTIL_getTime(&clockStart);
+            UTIL_waitForNextTick();
+            clockStart = UTIL_getTime();
 
             if (!cCompleted) {   /* still some time to do compression tests */
                 U32 nbLoops = 0;
@@ -260,8 +258,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         blockTable[blockNb].cSize = rSize;
                     }
                     nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                } while (UTIL_clockSpanMicro(clockStart) < clockLoop);
+                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
                     if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
                     totalCTime += clockSpan;
                     cCompleted = totalCTime>maxTime;
@@ -282,8 +280,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
 
             UTIL_sleepMilli(1); /* give processor time to other processes */
-            UTIL_waitForNextTick(ticksPerSecond);
-            UTIL_getTime(&clockStart);
+            UTIL_waitForNextTick();
+            clockStart = UTIL_getTime();
 
             if (!dCompleted) {
                 U32 nbLoops = 0;
@@ -300,8 +298,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         blockTable[blockNb].resSize = regenSize;
                     }
                     nbLoops++;
-                } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < DECOMP_MULT*clockLoop);
-                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+                } while (UTIL_clockSpanMicro(clockStart) < DECOMP_MULT*clockLoop);
+                {   U64 const clockSpan = UTIL_clockSpanMicro(clockStart);
                     if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
                     totalDTime += clockSpan;
                     dCompleted = totalDTime>(DECOMP_MULT*maxTime);
diff --git a/programs/util.h b/programs/util.h
index 5a69c55..1382cab 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -140,45 +140,116 @@ extern "C" {
 /*-****************************************
 *  Time functions
 ******************************************/
-#if (PLATFORM_POSIX_VERSION >= 1)
-#include 
-#include    /* times */
-   typedef U64 UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); }
-   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; }
-#elif defined(_WIN32)   /* Windows */
-   typedef LARGE_INTEGER UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); }
-   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; }
+#if defined(_WIN32)   /* Windows */
+    typedef LARGE_INTEGER UTIL_time_t;
+    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
+    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+    {
+        static LARGE_INTEGER ticksPerSecond;
+        static int init = 0;
+        if (!init) {
+            if (!QueryPerformanceFrequency(&ticksPerSecond))
+                fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n");
+            init = 1;
+        }
+        return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+    }
+    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+    {
+        static LARGE_INTEGER ticksPerSecond;
+        static int init = 0;
+        if (!init) {
+            if (!QueryPerformanceFrequency(&ticksPerSecond))
+                fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n");
+            init = 1;
+        }
+        return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+    }
+#elif defined(__APPLE__) && defined(__MACH__)
+    #include 
+    typedef U64 UTIL_time_t;
+    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+    {
+        static mach_timebase_info_data_t rate;
+        static int init = 0;
+        if (!init) {
+            mach_timebase_info(&rate);
+            init = 1;
+        }
+        return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
+    }
+    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+    {
+        static mach_timebase_info_data_t rate;
+        static int init = 0;
+        if (!init) {
+            mach_timebase_info(&rate);
+            init = 1;
+        }
+        return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
+    }
+#elif (PLATFORM_POSIX_VERSION >= 200112L)
+    #include 
+    typedef struct timespec UTIL_time_t;
+    UTIL_STATIC UTIL_time_t UTIL_getTime(void)
+    {
+        UTIL_time_t now;
+        if (clock_gettime(CLOCK_MONOTONIC, &now))
+            fprintf(stderr, "ERROR: Failed to get time\n");   /* we could also exit() */
+        return now;
+    }
+    UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
+    {
+        UTIL_time_t diff;
+        if (end.tv_nsec < begin.tv_nsec) {
+            diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
+            diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
+        } else {
+            diff.tv_sec = end.tv_sec - begin.tv_sec;
+            diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
+        }
+        return diff;
+    }
+    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
+    {
+        UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+        U64 micro = 0;
+        micro += 1000000ULL * diff.tv_sec;
+        micro += diff.tv_nsec / 1000ULL;
+        return micro;
+    }
+    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
+    {
+        UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
+        U64 nano = 0;
+        nano += 1000000000ULL * diff.tv_sec;
+        nano += diff.tv_nsec;
+        return nano;
+    }
 #else   /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
-   typedef clock_t UTIL_time_t;
-   UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; }
-   UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); }
-   UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
-   UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+    typedef clock_t UTIL_time_t;
+    UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
+    UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+    UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
 #endif
 
 
 /* returns time span in microseconds */
-UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond )
+UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart)
 {
-    UTIL_time_t clockEnd;
-    UTIL_getTime(&clockEnd);
-    return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
+    UTIL_time_t const clockEnd = UTIL_getTime();
+    return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 
 
-UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
+UTIL_STATIC void UTIL_waitForNextTick(void)
 {
-    UTIL_time_t clockStart, clockEnd;
-    UTIL_getTime(&clockStart);
+    UTIL_time_t const clockStart = UTIL_getTime();
+    UTIL_time_t clockEnd;
     do {
-        UTIL_getTime(&clockEnd);
-    } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
+        clockEnd = UTIL_getTime();
+    } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
 
 
-- 
cgit v0.12


From 708e2cbb60c7c074cbe9277388abc397403ef418 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 19 Oct 2017 16:39:40 -0700
Subject: simplified early exit when single solution

---
 lib/lz4opt.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 584dc97..1f8f580 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -245,11 +245,11 @@ static int LZ4HC_compress_optimal (
 
         if ((size_t)matches[match_num-1].len > sufficient_len) {
             /* good enough solution : immediate encoding */
-            best_mlen = matches[match_num-1].len;
-            best_off = matches[match_num-1].off;
-            cur = 0;
-            last_pos = 1;
-            goto encode;
+            int const firstML = (int)matches[match_num-1].len;
+            const BYTE* const matchPos = ip - matches[match_num-1].off;
+            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, (int)firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
+                return 0;  /* error */
+            continue;
         }
 
         /* set prices using matches at position = 0 */
-- 
cgit v0.12


From ac2ad52257defcc944471d0fdf3c6d68abebe904 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 19 Oct 2017 16:43:36 -0700
Subject: renamed last_pos into last_match_pos

---
 lib/lz4opt.h | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 1f8f580..e6a748c 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -197,7 +197,7 @@ LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
 
 #define SET_PRICE(pos, ml, offset, ll, cost)           \
 {                                                      \
-    while (last_pos < pos)  { opt[last_pos+1].price = 1<<30; last_pos++; } \
+    while (last_match_pos < pos)  { opt[last_match_pos+1].price = 1<<30; last_match_pos++; } \
     opt[pos].mlen = (int)ml;                           \
     opt[pos].off = (int)offset;                        \
     opt[pos].litlen = (int)ll;                         \
@@ -236,7 +236,7 @@ static int LZ4HC_compress_optimal (
     /* Main Loop */
     while (ip < mflimit) {
         size_t const llen = ip - anchor;
-        size_t last_pos = 0;
+        size_t last_match_pos = 0;
         size_t match_num, cur, best_mlen, best_off;
         memset(opt, 0, sizeof(LZ4HC_optimal_t));  /* memset only the first one */
 
@@ -259,14 +259,14 @@ static int LZ4HC_compress_optimal (
                 best_mlen = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
                 for ( ; mlen <= best_mlen ; mlen++) {
                     size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
-                    SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_pos and opt[pos] */
+                    SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_match_pos and opt[pos] */
         }   }   }
 
-        if (last_pos < MINMATCH) { ip++; continue; }  /* note : on clang at least, this test improves performance */
+        if (last_match_pos < MINMATCH) { ip++; continue; }  /* note : on clang at least, this test improves performance */
 
         /* check further positions */
         opt[0].mlen = opt[1].mlen = 1;
-        for (cur = 1; cur <= last_pos; cur++) {
+        for (cur = 1; cur <= last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
 
             /* establish baseline price if cur is literal */
@@ -285,17 +285,17 @@ static int LZ4HC_compress_optimal (
                 }
 
                 if (price < (size_t)opt[cur].price)
-                    SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price);   /* note : increases last_pos */
+                    SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price);   /* note : increases last_match_pos */
             }
 
-            if (cur == last_pos || curPtr >= mflimit) break;
+            if (cur == last_match_pos || curPtr >= mflimit) break;
 
             match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) {
                 /* immediate encoding */
                 best_mlen = matches[match_num-1].len;
                 best_off = matches[match_num-1].off;
-                last_pos = cur + 1;
+                last_match_pos = cur + 1;
                 goto encode;
             }
 
@@ -319,16 +319,16 @@ static int LZ4HC_compress_optimal (
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                         }
 
-                        if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) {
+                        if (cur + ml > last_match_pos || price < (size_t)opt[cur + ml].price) {
                             SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price);
             }   }   }   }
-        } /* for (cur = 1; cur <= last_pos; cur++) */
+        } /* for (cur = 1; cur <= last_match_pos; cur++) */
 
-        best_mlen = opt[last_pos].mlen;
-        best_off = opt[last_pos].off;
-        cur = last_pos - best_mlen;
+        best_mlen = opt[last_match_pos].mlen;
+        best_off = opt[last_match_pos].off;
+        cur = last_match_pos - best_mlen;
 
-encode: /* cur, last_pos, best_mlen, best_off must be set */
+encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         opt[0].mlen = 1;
         while (1) {  /* from end to beginning */
             size_t const ml = opt[cur].mlen;
@@ -343,7 +343,7 @@ encode: /* cur, last_pos, best_mlen, best_off must be set */
 
         /* encode all recorded sequences */
         cur = 0;
-        while (cur < last_pos) {
+        while (cur < last_match_pos) {
             int const ml = opt[cur].mlen;
             int const offset = opt[cur].off;
             if (ml == 1) { ip++; cur++; continue; }
-- 
cgit v0.12


From 6cec68de39345ac754d75b5ed694b7e3e28cc3ec Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 19 Oct 2017 16:47:25 -0700
Subject: added assert

---
 lib/lz4.c    | 8 ++++++++
 lib/lz4opt.h | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 19967f2..016e5c7 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -289,6 +289,14 @@ static const int LZ4_minLength = (MFLIMIT+1);
 /*-************************************
 *  Error detection
 **************************************/
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
+#  include 
+#else
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
+#endif
+
 #define LZ4_STATIC_ASSERT(c)   { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index e6a748c..69c48dc 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -262,7 +262,7 @@ static int LZ4HC_compress_optimal (
                     SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_match_pos and opt[pos] */
         }   }   }
 
-        if (last_match_pos < MINMATCH) { ip++; continue; }  /* note : on clang at least, this test improves performance */
+        assert(last_match_pos >= MINMATCH);
 
         /* check further positions */
         opt[0].mlen = opt[1].mlen = 1;
-- 
cgit v0.12


From 7bb0a617eeefacbbda15a0966319bed02ed68ac9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 11:00:10 -0700
Subject: simplified initial cost conditions

llen integrated in opt[]
---
 lib/lz4opt.h | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 69c48dc..f8eb50e 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -252,20 +252,29 @@ static int LZ4HC_compress_optimal (
             continue;
         }
 
-        /* set prices using matches at position = 0 */
+        /* set prices for first positions (literals) */
+        {   size_t rPos;
+            for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
+                int const cost = (int)LZ4HC_literalsPrice(llen + rPos);
+                opt[rPos].mlen = 1;
+                opt[rPos].off = 0;
+                opt[rPos].litlen = (int)(llen + rPos);
+                opt[rPos].price = cost;
+                DEBUGLOG(7, "rPos:%3u => cost:%3i (litlen=%i)",
+                            (U32)rPos, cost, opt[rPos].litlen);
+        }   }
+        /* set prices using matches found for rPos = 0 */
         {   size_t matchNb;
             for (matchNb = 0; matchNb < match_num; matchNb++) {
                 size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
                 best_mlen = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
                 for ( ; mlen <= best_mlen ; mlen++) {
-                    size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen);
+                    size_t const cost = LZ4HC_sequencePrice(llen, mlen);
                     SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_match_pos and opt[pos] */
         }   }   }
-
         assert(last_match_pos >= MINMATCH);
 
         /* check further positions */
-        opt[0].mlen = opt[1].mlen = 1;
         for (cur = 1; cur <= last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
 
@@ -274,11 +283,7 @@ static int LZ4HC_compress_optimal (
                 if (opt[cur-1].mlen == 1) {
                     /* no match at previous position */
                     litlen = opt[cur-1].litlen + 1;
-                    if (cur > litlen) {
-                        price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
-                    } else {
-                        price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
-                    }
+                    price = opt[cur-1].price - LZ4HC_literalsPrice(litlen-1) + LZ4HC_literalsPrice(litlen);
                 } else {
                     litlen = 1;
                     price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
@@ -313,7 +318,7 @@ static int LZ4HC_compress_optimal (
                             if (cur > ll)
                                 price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml);
                             else
-                                price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen);
+                                price = LZ4HC_sequencePrice(ll, ml);
                         } else {
                             ll = 0;
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
-- 
cgit v0.12


From c058753393063b4ec72de996b103327710f2c36b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 11:24:56 -0700
Subject: refactor variable matchnum

separate initial and iterative search
renamed nb_matches
---
 lib/lz4opt.h | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index f8eb50e..c7c95a8 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -197,7 +197,7 @@ LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
 
 #define SET_PRICE(pos, ml, offset, ll, cost)           \
 {                                                      \
-    while (last_match_pos < pos)  { opt[last_match_pos+1].price = 1<<30; last_match_pos++; } \
+    while (last_match_pos < pos) { opt[last_match_pos+1].price = 1<<30; last_match_pos++; } \
     opt[pos].mlen = (int)ml;                           \
     opt[pos].off = (int)offset;                        \
     opt[pos].litlen = (int)ll;                         \
@@ -237,16 +237,15 @@ static int LZ4HC_compress_optimal (
     while (ip < mflimit) {
         size_t const llen = ip - anchor;
         size_t last_match_pos = 0;
-        size_t match_num, cur, best_mlen, best_off;
-        memset(opt, 0, sizeof(LZ4HC_optimal_t));  /* memset only the first one */
+        size_t cur, best_mlen, best_off;
 
-        match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
-        if (!match_num) { ip++; continue; }
+        size_t const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+        if (!nb_matches_initial) { ip++; continue; }
 
-        if ((size_t)matches[match_num-1].len > sufficient_len) {
+        if ((size_t)matches[nb_matches_initial-1].len > sufficient_len) {
             /* good enough solution : immediate encoding */
-            int const firstML = (int)matches[match_num-1].len;
-            const BYTE* const matchPos = ip - matches[match_num-1].off;
+            int const firstML = (int)matches[nb_matches_initial-1].len;
+            const BYTE* const matchPos = ip - matches[nb_matches_initial-1].off;
             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, (int)firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
                 return 0;  /* error */
             continue;
@@ -265,7 +264,7 @@ static int LZ4HC_compress_optimal (
         }   }
         /* set prices using matches found for rPos = 0 */
         {   size_t matchNb;
-            for (matchNb = 0; matchNb < match_num; matchNb++) {
+            for (matchNb = 0; matchNb < nb_matches_initial; matchNb++) {
                 size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
                 best_mlen = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
                 for ( ; mlen <= best_mlen ; mlen++) {
@@ -277,6 +276,7 @@ static int LZ4HC_compress_optimal (
         /* check further positions */
         for (cur = 1; cur <= last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
+            size_t nb_matches;
 
             /* establish baseline price if cur is literal */
             {   size_t price, litlen;
@@ -295,18 +295,18 @@ static int LZ4HC_compress_optimal (
 
             if (cur == last_match_pos || curPtr >= mflimit) break;
 
-            match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) {
+            nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
                 /* immediate encoding */
-                best_mlen = matches[match_num-1].len;
-                best_off = matches[match_num-1].off;
+                best_mlen = matches[nb_matches-1].len;
+                best_off = matches[nb_matches-1].off;
                 last_match_pos = cur + 1;
                 goto encode;
             }
 
             /* set prices using matches at position = cur */
             {   size_t matchNb;
-                for (matchNb = 0; matchNb < match_num; matchNb++) {
+                for (matchNb = 0; matchNb < nb_matches; matchNb++) {
                     size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
                     best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
                                 (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur;
-- 
cgit v0.12


From fc879fe170d1644f8fd93017d53e44bc2b3cde04 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 11:32:15 -0700
Subject: lz4opt: refactor sequence reverse traversal

---
 lib/lz4opt.h | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index c7c95a8..7d487a7 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -334,17 +334,25 @@ static int LZ4HC_compress_optimal (
         cur = last_match_pos - best_mlen;
 
 encode: /* cur, last_match_pos, best_mlen, best_off must be set */
+        assert(cur < LZ4_OPT_NUM);
+        assert(last_match_pos >= 1);  /* == 1 when only one candidate */
         opt[0].mlen = 1;
-        while (1) {  /* from end to beginning */
-            size_t const ml = opt[cur].mlen;
-            int const offset = opt[cur].off;
-            opt[cur].mlen = (int)best_mlen;
-            opt[cur].off = (int)best_off;
-            best_mlen = ml;
-            best_off = offset;
-            if (ml > cur) break;   /* can this happen ? */
-            cur -= ml;
-        }
+        DEBUGLOG(6, "sequence reverse traversal");
+        {   int candidate_pos = (int)cur;
+            int selected_matchLength = (int)best_mlen;
+            int selected_offset = (int)best_off;
+            while (1) {  /* from end to beginning */
+                int const next_matchLength = opt[candidate_pos].mlen;
+                int const next_offset = opt[candidate_pos].off;
+                assert(next_matchLength > 0); /* note : can be 1, means literal */
+                opt[candidate_pos].mlen = selected_matchLength;
+                opt[candidate_pos].off = selected_offset;
+                DEBUGLOG(6, "rPos:%3i, matchLength:%3i", candidate_pos, selected_matchLength);
+                selected_matchLength = next_matchLength;
+                selected_offset = next_offset;
+                if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
+                candidate_pos -= next_matchLength;
+        }   }
 
         /* encode all recorded sequences */
         cur = 0;
@@ -352,7 +360,9 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
             int const ml = opt[cur].mlen;
             int const offset = opt[cur].off;
             if (ml == 1) { ip++; cur++; continue; }
+            assert(ml >= MINMATCH);
             cur += ml;
+            assert((offset >= 1) && (offset <=65535));
             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0;
         }
     }  /* while (ip < mflimit) */
-- 
cgit v0.12


From ee62faee085fe04ed657626a2320a5140989746c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 12:05:00 -0700
Subject: minor refactor

reduce variable scope
remove one macro usage
---
 lib/lz4opt.h | 63 +++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 7d487a7..48862f2 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -265,13 +265,18 @@ static int LZ4HC_compress_optimal (
         /* set prices using matches found for rPos = 0 */
         {   size_t matchNb;
             for (matchNb = 0; matchNb < nb_matches_initial; matchNb++) {
-                size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
-                best_mlen = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
-                for ( ; mlen <= best_mlen ; mlen++) {
+                int mlen = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
+                int const matchML = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
+                int const offset = matches[matchNb].off;
+                assert(matchML < LZ4_OPT_NUM);
+                for ( ; mlen <= matchML ; mlen++) {
                     size_t const cost = LZ4HC_sequencePrice(llen, mlen);
-                    SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost);   /* updates last_match_pos and opt[pos] */
+                    opt[mlen].mlen = mlen;
+                    opt[mlen].off = offset;
+                    opt[mlen].litlen = (int)llen;
+                    opt[mlen].price = (int)cost;
         }   }   }
-        assert(last_match_pos >= MINMATCH);
+        last_match_pos = matches[nb_matches_initial-1].len;
 
         /* check further positions */
         for (cur = 1; cur <= last_match_pos; cur++) {
@@ -308,17 +313,14 @@ static int LZ4HC_compress_optimal (
             {   size_t matchNb;
                 for (matchNb = 0; matchNb < nb_matches; matchNb++) {
                     size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
-                    best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
+                    size_t const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
                                 (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur;
 
-                    for ( ; ml <= best_mlen ; ml++) {
+                    for ( ; ml <= matchML ; ml++) {
                         size_t ll, price;
                         if (opt[cur].mlen == 1) {
                             ll = opt[cur].litlen;
-                            if (cur > ll)
-                                price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml);
-                            else
-                                price = LZ4HC_sequencePrice(ll, ml);
+                            price = ((cur > ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
                         } else {
                             ll = 0;
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
@@ -327,7 +329,7 @@ static int LZ4HC_compress_optimal (
                         if (cur + ml > last_match_pos || price < (size_t)opt[cur + ml].price) {
                             SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price);
             }   }   }   }
-        } /* for (cur = 1; cur <= last_match_pos; cur++) */
+        }  /* for (cur = 1; cur <= last_match_pos; cur++) */
 
         best_mlen = opt[last_match_pos].mlen;
         best_off = opt[last_match_pos].off;
@@ -337,7 +339,6 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         assert(cur < LZ4_OPT_NUM);
         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
         opt[0].mlen = 1;
-        DEBUGLOG(6, "sequence reverse traversal");
         {   int candidate_pos = (int)cur;
             int selected_matchLength = (int)best_mlen;
             int selected_offset = (int)best_off;
@@ -347,31 +348,37 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
                 assert(next_matchLength > 0); /* note : can be 1, means literal */
                 opt[candidate_pos].mlen = selected_matchLength;
                 opt[candidate_pos].off = selected_offset;
-                DEBUGLOG(6, "rPos:%3i, matchLength:%3i", candidate_pos, selected_matchLength);
                 selected_matchLength = next_matchLength;
                 selected_offset = next_offset;
                 if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
                 candidate_pos -= next_matchLength;
         }   }
 
-        /* encode all recorded sequences */
-        cur = 0;
-        while (cur < last_match_pos) {
-            int const ml = opt[cur].mlen;
-            int const offset = opt[cur].off;
-            if (ml == 1) { ip++; cur++; continue; }
-            assert(ml >= MINMATCH);
-            cur += ml;
-            assert((offset >= 1) && (offset <=65535));
-            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0;
-        }
+        /* encode all recorded sequences in order */
+        {   size_t rPos = 0;  /* relative position (to ip) */
+            while (rPos < last_match_pos) {
+                int const ml = opt[rPos].mlen;
+                int const offset = opt[rPos].off;
+                if (ml == 1) { ip++; rPos++; continue; }  /* literal */
+                rPos += ml;
+                assert(ml >= MINMATCH);
+                assert((offset >= 1) && (offset <=65535));
+                if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
+                    return 0;  /* error */
+        }   }
     }  /* while (ip < mflimit) */
 
     /* Encode Last Literals */
     {   int lastRun = (int)(iend - anchor);
-        if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;  /* Check output limit */
-        if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
-        else *op++ = (BYTE)(lastRun< (U32)maxOutputSize))
+            return 0;  /* Check output limit */
+        if (lastRun>=(int)RUN_MASK) {
+            *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255;
+            *op++ = (BYTE) lastRun;
+        } else *op++ = (BYTE)(lastRun<
Date: Fri, 20 Oct 2017 13:32:45 -0700
Subject: removed one macro usage

---
 lib/lz4opt.h | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 48862f2..488af90 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -284,7 +284,8 @@ static int LZ4HC_compress_optimal (
             size_t nb_matches;
 
             /* establish baseline price if cur is literal */
-            {   size_t price, litlen;
+            {   size_t price;
+                int litlen;
                 if (opt[cur-1].mlen == 1) {
                     /* no match at previous position */
                     litlen = opt[cur-1].litlen + 1;
@@ -293,13 +294,19 @@ static int LZ4HC_compress_optimal (
                     litlen = 1;
                     price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
                 }
-
-                if (price < (size_t)opt[cur].price)
-                    SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price);   /* note : increases last_match_pos */
+                if (price < (size_t)opt[cur].price) {
+                    opt[cur].mlen = 1;
+                    opt[cur].off = 0;
+                    opt[cur].litlen = litlen;
+                    opt[cur].price = (int)price;
+                }
             }
 
             if (cur == last_match_pos || curPtr >= mflimit) break;
 
+            //assert(cur+2 <= last_match_pos);
+            //assert(cur+3 <= last_match_pos);
+
             nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
                 /* immediate encoding */
-- 
cgit v0.12


From d813134619d2d2cae09d0f8bb1629ef1206f269d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 13:44:49 -0700
Subject: removed SET_PRICE macro

---
 lib/lz4opt.h | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 488af90..dd4e15d 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -195,16 +195,6 @@ LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
 }
 
 
-#define SET_PRICE(pos, ml, offset, ll, cost)           \
-{                                                      \
-    while (last_match_pos < pos) { opt[last_match_pos+1].price = 1<<30; last_match_pos++; } \
-    opt[pos].mlen = (int)ml;                           \
-    opt[pos].off = (int)offset;                        \
-    opt[pos].litlen = (int)ll;                         \
-    opt[pos].price = (int)cost;                        \
-}
-
-
 static int LZ4HC_compress_optimal (
     LZ4HC_CCtx_internal* ctx,
     const char* const source,
@@ -319,22 +309,29 @@ static int LZ4HC_compress_optimal (
             /* set prices using matches at position = cur */
             {   size_t matchNb;
                 for (matchNb = 0; matchNb < nb_matches; matchNb++) {
-                    size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH;
-                    size_t const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
-                                (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur;
+                    int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
+                    int const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
+                                matches[matchNb].len : (int)(LZ4_OPT_NUM - cur);
 
                     for ( ; ml <= matchML ; ml++) {
-                        size_t ll, price;
+                        int const pos = cur + ml;
+                        int const offset = matches[matchNb].off;
+                        size_t price;
+                        int ll;
                         if (opt[cur].mlen == 1) {
                             ll = opt[cur].litlen;
-                            price = ((cur > ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
+                            price = ((cur > (size_t)ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
                         } else {
                             ll = 0;
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                         }
 
-                        if (cur + ml > last_match_pos || price < (size_t)opt[cur + ml].price) {
-                            SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price);
+                        if ((size_t)pos > last_match_pos || price < (size_t)opt[pos].price) {
+                            while (last_match_pos < (size_t)pos) opt[++last_match_pos].price = 1<<30;
+                            opt[pos].mlen = ml;
+                            opt[pos].off = offset;
+                            opt[pos].litlen = ll;
+                            opt[pos].price = (int)price;
             }   }   }   }
         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
 
-- 
cgit v0.12


From fd6bd5107b81243952a4b2e0b3de901f45608b78 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 15:30:50 -0700
Subject: switched many types to int

---
 lib/lz4opt.h | 75 ++++++++++++++++++++++++++++++------------------------------
 1 file changed, 37 insertions(+), 38 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index dd4e15d..1b215b4 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -50,24 +50,24 @@ typedef struct {
 
 
 /* price in bytes */
-LZ4_FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
+LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen)
 {
-    size_t price = litlen;
-    if (litlen >= (size_t)RUN_MASK)
+    int price = litlen;
+    if (litlen >= (int)RUN_MASK)
         price += 1 + (litlen-RUN_MASK)/255;
     return price;
 }
 
 
 /* requires mlen >= MINMATCH */
-LZ4_FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
+LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 {
-    size_t price = 2 + 1; /* 16-bit offset + token */
+    int price = 2 + 1; /* 16-bit offset + token */
 
     price += LZ4HC_literalsPrice(litlen);
 
-    if (mlen >= (size_t)(ML_MASK+MINMATCH))
-        price+= 1 + (mlen-(ML_MASK+MINMATCH))/255;
+    if (mlen >= (int)(ML_MASK+MINMATCH))
+        price += 1 + (mlen-(ML_MASK+MINMATCH))/255;
 
     return price;
 }
@@ -224,57 +224,58 @@ static int LZ4HC_compress_optimal (
     ip++;
 
     /* Main Loop */
+    assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
     while (ip < mflimit) {
-        size_t const llen = ip - anchor;
-        size_t last_match_pos = 0;
-        size_t cur, best_mlen, best_off;
+        int const llen = (int)(ip - anchor);
+        int best_mlen, best_off;
+        int cur, last_match_pos = 0;
 
-        size_t const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+        int const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!nb_matches_initial) { ip++; continue; }
 
         if ((size_t)matches[nb_matches_initial-1].len > sufficient_len) {
             /* good enough solution : immediate encoding */
-            int const firstML = (int)matches[nb_matches_initial-1].len;
+            int const firstML = matches[nb_matches_initial-1].len;
             const BYTE* const matchPos = ip - matches[nb_matches_initial-1].off;
-            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, (int)firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
+            if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
                 return 0;  /* error */
             continue;
         }
 
         /* set prices for first positions (literals) */
-        {   size_t rPos;
+        {   int rPos;
             for (rPos = 0 ; rPos < MINMATCH ; rPos++) {
-                int const cost = (int)LZ4HC_literalsPrice(llen + rPos);
+                int const cost = LZ4HC_literalsPrice(llen + rPos);
                 opt[rPos].mlen = 1;
                 opt[rPos].off = 0;
-                opt[rPos].litlen = (int)(llen + rPos);
+                opt[rPos].litlen = llen + rPos;
                 opt[rPos].price = cost;
                 DEBUGLOG(7, "rPos:%3u => cost:%3i (litlen=%i)",
                             (U32)rPos, cost, opt[rPos].litlen);
         }   }
         /* set prices using matches found for rPos = 0 */
-        {   size_t matchNb;
+        {   int matchNb;
             for (matchNb = 0; matchNb < nb_matches_initial; matchNb++) {
                 int mlen = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
                 int const matchML = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
                 int const offset = matches[matchNb].off;
                 assert(matchML < LZ4_OPT_NUM);
                 for ( ; mlen <= matchML ; mlen++) {
-                    size_t const cost = LZ4HC_sequencePrice(llen, mlen);
+                    int const cost = LZ4HC_sequencePrice(llen, mlen);
                     opt[mlen].mlen = mlen;
                     opt[mlen].off = offset;
-                    opt[mlen].litlen = (int)llen;
-                    opt[mlen].price = (int)cost;
+                    opt[mlen].litlen = llen;
+                    opt[mlen].price = cost;
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
 
         /* check further positions */
         for (cur = 1; cur <= last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
-            size_t nb_matches;
+            int nb_matches;
 
             /* establish baseline price if cur is literal */
-            {   size_t price;
+            {   int price;
                 int litlen;
                 if (opt[cur-1].mlen == 1) {
                     /* no match at previous position */
@@ -284,19 +285,16 @@ static int LZ4HC_compress_optimal (
                     litlen = 1;
                     price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
                 }
-                if (price < (size_t)opt[cur].price) {
+                if (price < opt[cur].price) {
                     opt[cur].mlen = 1;
                     opt[cur].off = 0;
                     opt[cur].litlen = litlen;
-                    opt[cur].price = (int)price;
+                    opt[cur].price = price;
                 }
             }
 
             if (cur == last_match_pos || curPtr >= mflimit) break;
 
-            //assert(cur+2 <= last_match_pos);
-            //assert(cur+3 <= last_match_pos);
-
             nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
                 /* immediate encoding */
@@ -307,7 +305,7 @@ static int LZ4HC_compress_optimal (
             }
 
             /* set prices using matches at position = cur */
-            {   size_t matchNb;
+            {   int matchNb;
                 for (matchNb = 0; matchNb < nb_matches; matchNb++) {
                     int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
                     int const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
@@ -316,23 +314,24 @@ static int LZ4HC_compress_optimal (
                     for ( ; ml <= matchML ; ml++) {
                         int const pos = cur + ml;
                         int const offset = matches[matchNb].off;
-                        size_t price;
+                        int price;
                         int ll;
                         if (opt[cur].mlen == 1) {
                             ll = opt[cur].litlen;
-                            price = ((cur > (size_t)ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
+                            price = ((cur > ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
                         } else {
                             ll = 0;
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                         }
 
-                        if ((size_t)pos > last_match_pos || price < (size_t)opt[pos].price) {
-                            while (last_match_pos < (size_t)pos) opt[++last_match_pos].price = 1<<30;
+                        if (pos > last_match_pos || price < opt[pos].price) {
+                            while (last_match_pos < pos) opt[++last_match_pos].price = 1<<30;
                             opt[pos].mlen = ml;
                             opt[pos].off = offset;
                             opt[pos].litlen = ll;
-                            opt[pos].price = (int)price;
+                            opt[pos].price = price;
             }   }   }   }
+
         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
 
         best_mlen = opt[last_match_pos].mlen;
@@ -343,9 +342,9 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         assert(cur < LZ4_OPT_NUM);
         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
         opt[0].mlen = 1;
-        {   int candidate_pos = (int)cur;
-            int selected_matchLength = (int)best_mlen;
-            int selected_offset = (int)best_off;
+        {   int candidate_pos = cur;
+            int selected_matchLength = best_mlen;
+            int selected_offset = best_off;
             while (1) {  /* from end to beginning */
                 int const next_matchLength = opt[candidate_pos].mlen;
                 int const next_offset = opt[candidate_pos].off;
@@ -359,7 +358,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         }   }
 
         /* encode all recorded sequences in order */
-        {   size_t rPos = 0;  /* relative position (to ip) */
+        {   int rPos = 0;  /* relative position (to ip) */
             while (rPos < last_match_pos) {
                 int const ml = opt[rPos].mlen;
                 int const offset = opt[rPos].off;
@@ -377,7 +376,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         if ( (limit)
           && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
             return 0;  /* Check output limit */
-        if (lastRun>=(int)RUN_MASK) {
+        if (lastRun >= (int)RUN_MASK) {
             *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255;
-- 
cgit v0.12


From a12cdf00c3d0a0e3e7b2638d4a256f14b15c0072 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 20 Oct 2017 17:04:29 -0700
Subject: lz4opt: added hash chain search

---
 lib/lz4hc.c  | 27 ++++++++++++++++++---------
 lib/lz4hc.h  |  2 +-
 lib/lz4opt.h | 58 ++++++++++++++++++++++++++++++++++++++++++++--------------
 3 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 47474d3..adabd9c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -183,7 +183,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     const BYTE* const dictBase = hc4->dictBase;
     int const delta = (int)(ip-iLowLimit);
     int nbAttempts = maxNbAttempts;
-    reg_t const pattern = LZ4_read_ARCH(ip);
+    U32 const pattern = LZ4_read32(ip);
     U32 matchIndex;
     repeat_state_e repeat = rep_untested;
     size_t srcPatternLength = 0;
@@ -195,10 +195,16 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
         nbAttempts--;
+        int trace = 0;
+        if (nbAttempts==0) {
+            trace = 1;
+            DEBUGLOG(2, "reached max nb of attempts ! : %08X => %08X ",
+                pattern, HASH_FUNCTION(pattern));
+        }
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
             if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
-                if (LZ4_read32(matchPtr) == (U32)pattern) {
+                if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
     #if 0
                     /* more generic but unfortunately slower ... */
@@ -221,9 +227,9 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
             }
         } else {   /* matchIndex < dictLimit */
             const BYTE* const matchPtr = dictBase + matchIndex;
-            if (LZ4_read32(matchPtr) == (U32)pattern) {
+            if (LZ4_read32(matchPtr) == pattern) {
                 int mlt;
-                int back=0;
+                int back = 0;
                 const BYTE* vLimit = ip + (dictLimit - matchIndex);
                 if (vLimit > iHighLimit) vLimit = iHighLimit;
                 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
@@ -243,22 +249,25 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
             matchIndex -= nextOffset;
             if (1 && (nextOffset==1)) {
+                if (trace) DEBUGLOG(2, "check repeat mode : %u", repeat);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
-                    if (LZ4_read32(ip+4) == (U32)pattern) {  /* should check ip limit */
+                    if ((pattern & 0xFFFF) == (pattern >> 16)) {   /* is it enough ? */
                         repeat = rep_confirmed;
-                        srcPatternLength = LZ4HC_countPattern(ip+8, iHighLimit, pattern) + 8;
+                        srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4;
                     } else {
                         repeat = rep_not;
                 }   }
                 if ( (repeat == rep_confirmed)   /* proven repeated pattern (1-2-4) */
                   && (matchIndex >= dictLimit) ) {   /* same segment only */
                     const BYTE* const matchPtr = base + matchIndex;
-                    if (LZ4_read_ARCH(matchPtr) == pattern) {  /* good candidate */
+                    if (trace) DEBUGLOG(2, "search direct pattern position");
+                    if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
-                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, (U32)pattern);
+                        size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
+                        if (trace) DEBUGLOG(2, "good start position (match == pattern)");
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
@@ -633,7 +642,7 @@ static int LZ4HC_getSearchNum(int compressionLevel)
     switch (compressionLevel) {
         default: return 0; /* unused */
         case 11: return 128;
-        case 12: return 1<<10;
+        case 12: return 1<<13;
     }
 }
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 66d5636..a9cefbb 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -129,7 +129,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
  * They are exposed to allow static allocation of `LZ4_streamHC_t`.
  * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
  **************************************/
-#define LZ4HC_DICTIONARY_LOGSIZE 17   /* because of btopt, hc would only need 16 */
+#define LZ4HC_DICTIONARY_LOGSIZE 17   /* due to btree, hc would only need 16 */
 #define LZ4HC_MAXD (1<searchNum);
+    if (matchLength < MINMATCH) return 0;
+    assert(matches != NULL);
+    matches[0].len = matchLength;
+    matches[0].off = (int)(ip-matchPtr);
+    (void)fullUpdate;
+    return 1;
+}
+
+
 static int LZ4HC_compress_optimal (
     LZ4HC_CCtx_internal* ctx,
     const char* const source,
-    char* dest,
+    char* dst,
     int inputSize,
-    int maxOutputSize,
+    int dstCapacity,
     limitedOutput_directive limit,
     size_t sufficient_len,
     const int fullUpdate
@@ -214,8 +241,8 @@ static int LZ4HC_compress_optimal (
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimit = iend - MFLIMIT;
     const BYTE* const matchlimit = (iend - LASTLITERALS);
-    BYTE* op = (BYTE*) dest;
-    BYTE* const oend = op + maxOutputSize;
+    BYTE* op = (BYTE*) dst;
+    BYTE* const oend = op + dstCapacity;
 
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
@@ -230,7 +257,8 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        int const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+        //int const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+        int const nb_matches_initial = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!nb_matches_initial) { ip++; continue; }
 
         if ((size_t)matches[nb_matches_initial-1].len > sufficient_len) {
@@ -274,7 +302,9 @@ static int LZ4HC_compress_optimal (
             const BYTE* const curPtr = ip + cur;
             int nb_matches;
 
-            /* establish baseline price if cur is literal */
+            /* establish baseline price for cur as a literal.
+             * fixes unused positions (between 2 matches)
+             * and inefficient match stack */
             {   int price;
                 int litlen;
                 if (opt[cur-1].mlen == 1) {
@@ -295,7 +325,8 @@ static int LZ4HC_compress_optimal (
 
             if (cur == last_match_pos || curPtr >= mflimit) break;
 
-            nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
                 /* immediate encoding */
                 best_mlen = matches[nb_matches-1].len;
@@ -307,9 +338,9 @@ static int LZ4HC_compress_optimal (
             /* set prices using matches at position = cur */
             {   int matchNb;
                 for (matchNb = 0; matchNb < nb_matches; matchNb++) {
-                    int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
                     int const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
                                 matches[matchNb].len : (int)(LZ4_OPT_NUM - cur);
+                    int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
 
                     for ( ; ml <= matchML ; ml++) {
                         int const pos = cur + ml;
@@ -341,19 +372,18 @@ static int LZ4HC_compress_optimal (
 encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         assert(cur < LZ4_OPT_NUM);
         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
-        opt[0].mlen = 1;
         {   int candidate_pos = cur;
             int selected_matchLength = best_mlen;
             int selected_offset = best_off;
             while (1) {  /* from end to beginning */
-                int const next_matchLength = opt[candidate_pos].mlen;
+                int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
                 int const next_offset = opt[candidate_pos].off;
-                assert(next_matchLength > 0); /* note : can be 1, means literal */
                 opt[candidate_pos].mlen = selected_matchLength;
                 opt[candidate_pos].off = selected_offset;
                 selected_matchLength = next_matchLength;
                 selected_offset = next_offset;
                 if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */
+                assert(next_matchLength > 0);  /* can be 1, means literal */
                 candidate_pos -= next_matchLength;
         }   }
 
@@ -362,7 +392,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
             while (rPos < last_match_pos) {
                 int const ml = opt[rPos].mlen;
                 int const offset = opt[rPos].off;
-                if (ml == 1) { ip++; rPos++; continue; }  /* literal */
+                if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
                 rPos += ml;
                 assert(ml >= MINMATCH);
                 assert((offset >= 1) && (offset <=65535));
@@ -374,7 +404,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
     /* Encode Last Literals */
     {   int lastRun = (int)(iend - anchor);
         if ( (limit)
-          && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
+          && (((char*)op - dst) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)dstCapacity))
             return 0;  /* Check output limit */
         if (lastRun >= (int)RUN_MASK) {
             *op++=(RUN_MASK<
Date: Wed, 25 Oct 2017 07:07:08 +0200
Subject: added hash chain with conditional length

not a success yet
---
 lib/lz4hc.c  | 9 ---------
 lib/lz4opt.h | 3 ++-
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index adabd9c..4532ba3 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -195,12 +195,6 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
         nbAttempts--;
-        int trace = 0;
-        if (nbAttempts==0) {
-            trace = 1;
-            DEBUGLOG(2, "reached max nb of attempts ! : %08X => %08X ",
-                pattern, HASH_FUNCTION(pattern));
-        }
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
             if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) {
@@ -249,7 +243,6 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
             matchIndex -= nextOffset;
             if (1 && (nextOffset==1)) {
-                if (trace) DEBUGLOG(2, "check repeat mode : %u", repeat);
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ((pattern & 0xFFFF) == (pattern >> 16)) {   /* is it enough ? */
@@ -261,13 +254,11 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                 if ( (repeat == rep_confirmed)   /* proven repeated pattern (1-2-4) */
                   && (matchIndex >= dictLimit) ) {   /* same segment only */
                     const BYTE* const matchPtr = base + matchIndex;
-                    if (trace) DEBUGLOG(2, "search direct pattern position");
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
                         size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern);
                         const BYTE* const maxLowPtr = (lowPrefixPtr + MAX_DISTANCE >= ip) ? lowPrefixPtr : ip - MAX_DISTANCE;
                         size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, maxLowPtr, pattern);
                         size_t const currentSegmentLength = backLength + forwardPatternLength;
-                        if (trace) DEBUGLOG(2, "good start position (match == pattern)");
 
                         if ( (currentSegmentLength >= srcPatternLength)   /* current pattern segment large enough to contain full srcPatternLength */
                           && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index ef7b725..bd956ee 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -213,7 +213,7 @@ LZ4_FORCE_INLINE int LZ4HC_HashChain_GetAllMatches (
 {
     const BYTE* matchPtr;
     int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, ctx->searchNum);
-    if (matchLength < MINMATCH) return 0;
+    if ((size_t)matchLength <= best_mlen) return 0;
     assert(matches != NULL);
     matches[0].len = matchLength;
     matches[0].off = (int)(ip-matchPtr);
@@ -327,6 +327,7 @@ static int LZ4HC_compress_optimal (
 
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur + 1, matches, fullUpdate);   /* only works if last_match_pos is really the last match pos */
             if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
                 /* immediate encoding */
                 best_mlen = matches[nb_matches-1].len;
-- 
cgit v0.12


From a31b7058cb97e4393da55e78a77a1c6f0c9ae038 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 25 Oct 2017 10:10:53 +0200
Subject: small modification of lz4 decoder to shortcut common case (short
 branch).

---
 lib/lz4.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/lib/lz4.c b/lib/lz4.c
index 179408d..e0a961f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1180,6 +1180,22 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         /* get literal length */
         unsigned const token = *ip++;
+
+        if (1)
+        if (ip + 14 + 2 <= iend)
+        if ((token < 15*16) & ((token & 0xF) <= 12)) {
+            size_t const ll = token >> ML_BITS;
+            size_t const off = LZ4_readLE16(ip+ll);   /* check input validity */
+            if (off >= 16) {
+                size_t const ml = (token & 0xF) + MINMATCH;
+                DEBUGLOG(2, "rest:%u, ll:%2u, ml:%2u, off:%u",
+                    (U32)(oend-op), (U32)ll, (U32)ml, (U32)off);
+                memcpy(op, ip, 16); op += ll; ip += ll + 2 /* offset */;
+                memcpy(op, op - off, 16); op += ml;
+                continue;
+            }
+        }
+
         if ((length=(token>>ML_BITS)) == RUN_MASK) {
             unsigned s;
             do {
-- 
cgit v0.12


From 63a7f34fee5e2ecac452aee3731b7390a1eb8328 Mon Sep 17 00:00:00 2001
From: mikir 
Date: Mon, 30 Oct 2017 13:44:24 +0100
Subject: Separated visibility from LZ4LIB_API macro.

---
 lib/lz4.h | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index d284d63..509d942 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -72,20 +72,24 @@ extern "C" {
 /*
 *  LZ4_DLL_EXPORT :
 *  Enable exporting of functions when building a Windows DLL
-*  LZ4LIB_API :
+*  LZ4LIB_VISIBILITY :
 *  Control library symbols visibility.
 */
+#ifndef LZ4LIB_VISIBILITY
+#  if defined(__GNUC__) && (__GNUC__ >= 4)
+#    define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
+#  else
+#    define LZ4LIB_VISIBILITY
+#  endif
+#endif
 #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
-#  define LZ4LIB_API __declspec(dllexport)
+#  define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
 #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
-#  define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
-#elif defined(__GNUC__) && (__GNUC__ >= 4)
-#  define LZ4LIB_API __attribute__ ((__visibility__ ("default")))
+#  define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
 #else
-#  define LZ4LIB_API
+#  define LZ4LIB_API LZ4LIB_VISIBILITY
 #endif
 
-
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    8    /* for new (non-breaking) interface capabilities */
-- 
cgit v0.12


From e0914ff70c4a8b94d900deeea44c7cd1e9ac4a07 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 30 Oct 2017 16:07:15 -0700
Subject: more complete shortcut - passes tests

---
 lib/lz4.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e0a961f..10f8d55 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1146,7 +1146,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
                  int partialDecoding,    /* full, partial */
                  int targetOutputSize,   /* only used if partialDecoding==partial */
                  int dict,               /* noDict, withPrefix64k, usingExtDict */
-                 const BYTE* const lowPrefix,  /* == dst when no prefix */
+                 const BYTE* const lowPrefix,  /* always <= dst, == dst when no prefix */
                  const BYTE* const dictStart,  /* only if dict==usingExtDict */
                  const size_t dictSize         /* note : = 0 if noDict */
                  )
@@ -1168,7 +1168,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
 
     /* Special cases */
-    if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                        /* targetOutputSize too high => decode everything */
+    if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT;                      /* targetOutputSize too high => just decode everything */
     if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1;  /* Empty output buffer */
     if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
 
@@ -1178,24 +1178,25 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         const BYTE* match;
         size_t offset;
 
-        /* get literal length */
         unsigned const token = *ip++;
 
-        if (1)
-        if (ip + 14 + 2 <= iend)
-        if ((token < 15*16) & ((token & 0xF) <= 12)) {
+        /* shortcut for common case */
+        /* this shortcut was tested on x86 and x64, where it improves decoding speed.
+         * it has not yet been benchmarked on ARM, Power, mips, etc. */
+        if (((ip + 14 + 2 <= iend) & (op + 14 + 16 <= oend))
+          & ((token < 15*16) & ((token & 0xF) <= 12)) ) {
             size_t const ll = token >> ML_BITS;
-            size_t const off = LZ4_readLE16(ip+ll);   /* check input validity */
-            if (off >= 16) {
+            size_t const off = LZ4_readLE16(ip+ll);
+            const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
+            if ((off >= 16) & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & 0xF) + MINMATCH;
-                DEBUGLOG(2, "rest:%u, ll:%2u, ml:%2u, off:%u",
-                    (U32)(oend-op), (U32)ll, (U32)ml, (U32)off);
-                memcpy(op, ip, 16); op += ll; ip += ll + 2 /* offset */;
-                memcpy(op, op - off, 16); op += ml;
+                memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
+                memcpy(op, matchPtr, 16); op += ml;
                 continue;
             }
         }
 
+        /* decode literal length */
         if ((length=(token>>ML_BITS)) == RUN_MASK) {
             unsigned s;
             do {
-- 
cgit v0.12


From ab4bd93f59dce5d116f1c95373a64cc39ccd6990 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 30 Oct 2017 16:10:25 -0700
Subject: fixed minor initialization warning

---
 lib/lz4opt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index bd956ee..2daf17e 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -211,7 +211,7 @@ LZ4_FORCE_INLINE int LZ4HC_HashChain_GetAllMatches (
                         const BYTE* const ip, const BYTE* const iHighLimit,
                         size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
 {
-    const BYTE* matchPtr;
+    const BYTE* matchPtr = NULL;
     int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, ctx->searchNum);
     if ((size_t)matchLength <= best_mlen) return 0;
     assert(matches != NULL);
-- 
cgit v0.12


From 931c5c20d0d87e706e3dd9bf57e089500455c987 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 30 Oct 2017 17:47:54 -0700
Subject: fixed minor overflow mistake in optimal parser

saving 20 bytes on calgary.tar
---
 lib/lz4opt.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 2daf17e..40592df 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -328,7 +328,10 @@ static int LZ4HC_compress_optimal (
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur + 1, matches, fullUpdate);   /* only works if last_match_pos is really the last match pos */
-            if ((nb_matches > 0) && (size_t)matches[nb_matches-1].len > sufficient_len) {
+            if (!nb_matches) continue;
+
+            if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
+              || (matches[nb_matches-1].len + cur >= LZ4_OPT_NUM) ) {
                 /* immediate encoding */
                 best_mlen = matches[nb_matches-1].len;
                 best_off = matches[nb_matches-1].off;
@@ -357,6 +360,7 @@ static int LZ4HC_compress_optimal (
                         }
 
                         if (pos > last_match_pos || price < opt[pos].price) {
+                            assert(pos < LZ4_OPT_NUM);
                             while (last_match_pos < pos) opt[++last_match_pos].price = 1<<30;
                             opt[pos].mlen = ml;
                             opt[pos].off = offset;
-- 
cgit v0.12


From 3f173052aef1d0f1503bd986cc871e59ef56cadd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 31 Oct 2017 11:49:57 -0700
Subject: added comments, as suggested by @terrelln

---
 lib/lz4.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 10f8d55..b036d98 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1180,15 +1180,16 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
 
         unsigned const token = *ip++;
 
-        /* shortcut for common case */
-        /* this shortcut was tested on x86 and x64, where it improves decoding speed.
+        /* shortcut for common case :
+         * in most circumstances, we expect to decode small matches (<= 16 bytes) separated by few literals (<= 14 bytes).
+         * this shortcut was tested on x86 and x64, where it improves decoding speed.
          * it has not yet been benchmarked on ARM, Power, mips, etc. */
         if (((ip + 14 + 2 <= iend) & (op + 14 + 16 <= oend))
-          & ((token < 15*16) & ((token & 0xF) <= 12)) ) {
+          & ((token < (15<> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 16) & (matchPtr >= lowPrefix)) {
+            if ((off >= 16) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & 0xF) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
                 memcpy(op, matchPtr, 16); op += ml;
-- 
cgit v0.12


From ace334a4c94927d97933888b343a2f435bbbc7fa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 31 Oct 2017 12:22:15 -0700
Subject: minor : coding style : use ML_MASK constant

---
 lib/lz4.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index b036d98..34d8c40 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1185,12 +1185,12 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
          * it has not yet been benchmarked on ARM, Power, mips, etc. */
         if (((ip + 14 + 2 <= iend) & (op + 14 + 16 <= oend))
-          & ((token < (15<> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
             if ((off >= 16) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
-                size_t const ml = (token & 0xF) + MINMATCH;
+                size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
                 memcpy(op, matchPtr, 16); op += ml;
                 continue;
-- 
cgit v0.12


From 9378f76e4186e7b4f33a79451b22afd5b6344c8d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 31 Oct 2017 14:20:25 -0700
Subject: extended shortcut match length to 18

---
 lib/lz4.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 34d8c40..1504790 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1181,18 +1181,19 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         unsigned const token = *ip++;
 
         /* shortcut for common case :
-         * in most circumstances, we expect to decode small matches (<= 16 bytes) separated by few literals (<= 14 bytes).
+         * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes).
          * this shortcut was tested on x86 and x64, where it improves decoding speed.
          * it has not yet been benchmarked on ARM, Power, mips, etc. */
-        if (((ip + 14 + 2 <= iend) & (op + 14 + 16 <= oend))
-          & ((token < (15<> ML_BITS;
             size_t const off = LZ4_readLE16(ip+ll);
             const BYTE* const matchPtr = op + ll - off;  /* pointer underflow risk ? */
-            if ((off >= 16) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
+            if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) {
                 size_t const ml = (token & ML_MASK) + MINMATCH;
                 memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/;
-                memcpy(op, matchPtr, 16); op += ml;
+                memcpy(op, matchPtr, 18); op += ml;
                 continue;
             }
         }
-- 
cgit v0.12


From a5731d6b266170f7bf0893c833af680d929f5616 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 31 Oct 2017 15:51:56 -0700
Subject: minor change, to help store forwarding

in a marginal case (offset==4)
---
 lib/lz4.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 1504790..5efcbc0 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1160,8 +1160,8 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
     BYTE* oexit = op + targetOutputSize;
 
     const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
-    const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};
-    const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
+    const unsigned inc32table[8] = {0, 1, 2,  1,  0,  4, 4, 4};
+    const int      dec64table[8] = {0, 0, 0, -1, -4,  1, 2, 3};
 
     const int safeDecode = (endOnInput==endOnInputSize);
     const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
@@ -1276,14 +1276,13 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
         /* copy match within block */
         cpy = op + length;
         if (unlikely(offset<8)) {
-            const int dec64 = dec64table[offset];
             op[0] = match[0];
             op[1] = match[1];
             op[2] = match[2];
             op[3] = match[3];
-            match += dec32table[offset];
+            match += inc32table[offset];
             memcpy(op+4, match, 4);
-            match -= dec64;
+            match -= dec64table[offset];
         } else { LZ4_copy8(op, match); match+=8; }
         op += 8;
 
@@ -1300,7 +1299,7 @@ LZ4_FORCE_INLINE int LZ4_decompress_generic(
             LZ4_copy8(op, match);
             if (length>16) LZ4_wildCopy(op+8, match+8, cpy);
         }
-        op=cpy;   /* correction */
+        op = cpy;   /* correction */
     }
 
     /* end of decoding */
-- 
cgit v0.12


From 0ff4df1b941145d5b728b4b4a936391537b8ea11 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 13:44:57 -0700
Subject: changed strategy : opt[] path is complete after each match

previous strategy would leave a few "bad choices"
on the ground they would be fixed later,
but that requires passing through each position to make the fix
and cannot give the end position of the last useful match.
---
 lib/lz4hc.c  | 21 +++++++++-----
 lib/lz4opt.h | 90 ++++++++++++++++++++++++++++++++++++++----------------------
 2 files changed, 71 insertions(+), 40 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 4532ba3..eb31a9c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -342,10 +342,6 @@ typedef enum {
     limitedDestSize = 2,
 } limitedOutput_directive;
 
-#ifndef LZ4HC_DEBUG
-#  define LZ4HC_DEBUG 0
-#endif
-
 /* LZ4HC_encodeSequence() :
  * @return : 0 if ok,
  *           1 if buffer issue detected */
@@ -361,9 +357,19 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     size_t length;
     BYTE* const token = (*op)++;
 
-#if LZ4HC_DEBUG
-    printf("literal : %u  --  match : %u  --  offset : %u\n",
-           (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
+    static const BYTE* start = NULL;
+    static U32 totalCost = 0;
+    U32 const ll = (U32)(*ip - *anchor);
+    U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
+    U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
+    U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
+    if (start==NULL) start = *anchor;  /* only works for single segment */
+    totalCost += cost;
+    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u/%7u",
+                (U32)(*anchor - start),
+                (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
+                cost, totalCost);
 #endif
 
     /* Encode Literal length */
@@ -386,6 +392,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
 
     /* Encode MatchLength */
+    assert(matchLength >= MINMATCH);
     length = (size_t)(matchLength - MINMATCH);
     if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1;   /* Check output limit */
     if (length >= ML_MASK) {
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 40592df..8a223ec 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -233,14 +233,14 @@ static int LZ4HC_compress_optimal (
     const int fullUpdate
     )
 {
-    LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];   /* this uses a bit too much stack memory to my taste ... */
+    LZ4HC_optimal_t opt[LZ4_OPT_NUM + 3];   /* this uses a bit too much stack memory to my taste ... */
     LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
     const BYTE* const iend = ip + inputSize;
     const BYTE* const mflimit = iend - MFLIMIT;
-    const BYTE* const matchlimit = (iend - LASTLITERALS);
+    const BYTE* const matchlimit = iend - LASTLITERALS;
     BYTE* op = (BYTE*) dst;
     BYTE* const oend = op + dstCapacity;
 
@@ -278,8 +278,8 @@ static int LZ4HC_compress_optimal (
                 opt[rPos].off = 0;
                 opt[rPos].litlen = llen + rPos;
                 opt[rPos].price = cost;
-                DEBUGLOG(7, "rPos:%3u => cost:%3i (litlen=%i)",
-                            (U32)rPos, cost, opt[rPos].litlen);
+                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                            rPos, cost, opt[rPos].litlen);
         }   }
         /* set prices using matches found for rPos = 0 */
         {   int matchNb;
@@ -294,36 +294,26 @@ static int LZ4HC_compress_optimal (
                     opt[mlen].off = offset;
                     opt[mlen].litlen = llen;
                     opt[mlen].price = cost;
+                    DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
+                                mlen, cost, mlen);
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
+        {   int addLit;
+            for (addLit = 1; addLit <= 2; addLit ++) {
+                opt[last_match_pos+addLit].mlen = 1; /* literal */
+                opt[last_match_pos+addLit].off = 0;
+                opt[last_match_pos+addLit].litlen = addLit;
+                opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
+                            last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+        }   }
 
         /* check further positions */
-        for (cur = 1; cur <= last_match_pos; cur++) {
+        for (cur = 1; cur < last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
             int nb_matches;
 
-            /* establish baseline price for cur as a literal.
-             * fixes unused positions (between 2 matches)
-             * and inefficient match stack */
-            {   int price;
-                int litlen;
-                if (opt[cur-1].mlen == 1) {
-                    /* no match at previous position */
-                    litlen = opt[cur-1].litlen + 1;
-                    price = opt[cur-1].price - LZ4HC_literalsPrice(litlen-1) + LZ4HC_literalsPrice(litlen);
-                } else {
-                    litlen = 1;
-                    price = opt[cur - 1].price + LZ4HC_literalsPrice(1);
-                }
-                if (price < opt[cur].price) {
-                    opt[cur].mlen = 1;
-                    opt[cur].off = 0;
-                    opt[cur].litlen = litlen;
-                    opt[cur].price = price;
-                }
-            }
-
-            if (cur == last_match_pos || curPtr >= mflimit) break;
+            if (curPtr >= mflimit) break;
 
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
@@ -339,11 +329,26 @@ static int LZ4HC_compress_optimal (
                 goto encode;
             }
 
+            /* before first match : set price with literals at beginning */
+            {   int const baseLitlen = opt[cur].litlen;
+                int litlen;
+                for (litlen = 1; litlen < MINMATCH; litlen++) {
+                    int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen);
+                    int const pos = cur + litlen;
+                    if (price < opt[pos].price) {
+                        opt[pos].mlen = 1; /* literal */
+                        opt[pos].off = 0;
+                        opt[pos].litlen = baseLitlen+litlen;
+                        opt[pos].price = price;
+                        DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
+                                    pos, price, opt[pos].litlen);
+            }   }   }
+
             /* set prices using matches at position = cur */
             {   int matchNb;
+                assert(cur + matches[nb_matches-1].len < LZ4_OPT_NUM);
                 for (matchNb = 0; matchNb < nb_matches; matchNb++) {
-                    int const matchML = (cur + matches[matchNb].len < LZ4_OPT_NUM) ?
-                                matches[matchNb].len : (int)(LZ4_OPT_NUM - cur);
+                    int const matchML = matches[matchNb].len;
                     int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
 
                     for ( ; ml <= matchML ; ml++) {
@@ -351,23 +356,39 @@ static int LZ4HC_compress_optimal (
                         int const offset = matches[matchNb].off;
                         int price;
                         int ll;
+                        DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
+                                    pos, last_match_pos);
                         if (opt[cur].mlen == 1) {
                             ll = opt[cur].litlen;
-                            price = ((cur > ll) ? opt[cur - ll].price : 0) + LZ4HC_sequencePrice(ll, ml);
+                            price = ((cur > ll) ? opt[cur - ll].price : 0)
+                                  + LZ4HC_sequencePrice(ll, ml);
                         } else {
                             ll = 0;
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                         }
 
-                        if (pos > last_match_pos || price < opt[pos].price) {
+                        if (pos > last_match_pos+2 || price <= opt[pos].price) {
+                            DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
+                                        pos, price, ml);
                             assert(pos < LZ4_OPT_NUM);
-                            while (last_match_pos < pos) opt[++last_match_pos].price = 1<<30;
+                            if ( (matchNb == nb_matches-1)  /* last match */
+                              && (ml == matchML)  /* last post of last match */
+                              && (last_match_pos < pos) )
+                                last_match_pos = pos;
                             opt[pos].mlen = ml;
                             opt[pos].off = offset;
                             opt[pos].litlen = ll;
                             opt[pos].price = price;
             }   }   }   }
-
+            /* complete following positions with literals */
+            {   int addLit;
+                for (addLit = 1; addLit <= 2; addLit ++) {
+                    opt[last_match_pos+addLit].mlen = 1; /* literal */
+                    opt[last_match_pos+addLit].off = 0;
+                    opt[last_match_pos+addLit].litlen = addLit;
+                    opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                    DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
+            }   }
         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
 
         best_mlen = opt[last_match_pos].mlen;
@@ -377,12 +398,15 @@ static int LZ4HC_compress_optimal (
 encode: /* cur, last_match_pos, best_mlen, best_off must be set */
         assert(cur < LZ4_OPT_NUM);
         assert(last_match_pos >= 1);  /* == 1 when only one candidate */
+        DEBUGLOG(6, "reverse traversal, looking for shortest path")
+        DEBUGLOG(6, "last_match_pos = %i", last_match_pos);
         {   int candidate_pos = cur;
             int selected_matchLength = best_mlen;
             int selected_offset = best_off;
             while (1) {  /* from end to beginning */
                 int const next_matchLength = opt[candidate_pos].mlen;  /* can be 1, means literal */
                 int const next_offset = opt[candidate_pos].off;
+                DEBUGLOG(6, "pos %i: sequence length %i", candidate_pos, selected_matchLength);
                 opt[candidate_pos].mlen = selected_matchLength;
                 opt[candidate_pos].off = selected_offset;
                 selected_matchLength = next_matchLength;
-- 
cgit v0.12


From 8e16eb0cd164071d3fc4c18e0f10527487649e93 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 14:53:06 -0700
Subject: fixed last lost bytes in maximal mode

even gained 2 bytes on calgary.tar...
added conditional traces `g_debuglog_enable`
---
 lib/lz4.c    | 12 +++++++-----
 lib/lz4hc.c  | 13 +++++++++----
 lib/lz4opt.h |  7 ++++---
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index e3e9172..e21822d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -324,11 +324,12 @@ static const int LZ4_minLength = (MFLIMIT+1);
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2)
 #  include 
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=LZ4_DEBUG) {                   \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
+static int g_debuglog_enable = 1;
+#  define DEBUGLOG(l, ...) {                                  \
+                if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
+                    fprintf(stderr, __FILE__ ": ");           \
+                    fprintf(stderr, __VA_ARGS__);             \
+                    fprintf(stderr, " \n");                   \
             }   }
 #else
 #  define DEBUGLOG(l, ...)      {}    /* disabled */
@@ -978,6 +979,7 @@ LZ4_stream_t* LZ4_createStream(void)
 
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
+    DEBUGLOG(4, "LZ4_resetStream");
     MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
 }
 
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index eb31a9c..884f5d7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -188,12 +188,15 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     repeat_state_e repeat = rep_untested;
     size_t srcPatternLength = 0;
 
-
+    DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch");
     /* First Match */
     LZ4HC_Insert(hc4, ip);
     matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+    DEBUGLOG(7, "First match at index %u / %u (lowLimit)",
+                matchIndex, lowLimit);
 
     while ((matchIndex>=lowLimit) && (nbAttempts)) {
+        DEBUGLOG(7, "remaining attempts : %i", nbAttempts);
         nbAttempts--;
         if (matchIndex >= dictLimit) {
             const BYTE* const matchPtr = base + matchIndex;
@@ -360,16 +363,18 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
     static const BYTE* start = NULL;
     static U32 totalCost = 0;
+    U32 const pos = (U32)(*anchor - start);
     U32 const ll = (U32)(*ip - *anchor);
     U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
-    totalCost += cost;
-    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u/%7u",
-                (U32)(*anchor - start),
+    //g_debuglog_enable = (pos >= 112705) & (pos <= 112760);
+    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u / %u",
+                pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
                 cost, totalCost);
+    totalCost += cost;
 #endif
 
     /* Encode Literal length */
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 8a223ec..25ceaad 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -299,7 +299,7 @@ static int LZ4HC_compress_optimal (
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
         {   int addLit;
-            for (addLit = 1; addLit <= 2; addLit ++) {
+            for (addLit = 1; addLit <= 3; addLit ++) {
                 opt[last_match_pos+addLit].mlen = 1; /* literal */
                 opt[last_match_pos+addLit].off = 0;
                 opt[last_match_pos+addLit].litlen = addLit;
@@ -315,6 +315,7 @@ static int LZ4HC_compress_optimal (
 
             if (curPtr >= mflimit) break;
 
+            DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur + 1, matches, fullUpdate);   /* only works if last_match_pos is really the last match pos */
@@ -367,7 +368,7 @@ static int LZ4HC_compress_optimal (
                             price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                         }
 
-                        if (pos > last_match_pos+2 || price <= opt[pos].price) {
+                        if (pos > last_match_pos+3 || price <= opt[pos].price) {
                             DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
                                         pos, price, ml);
                             assert(pos < LZ4_OPT_NUM);
@@ -382,7 +383,7 @@ static int LZ4HC_compress_optimal (
             }   }   }   }
             /* complete following positions with literals */
             {   int addLit;
-                for (addLit = 1; addLit <= 2; addLit ++) {
+                for (addLit = 1; addLit <= 3; addLit ++) {
                     opt[last_match_pos+addLit].mlen = 1; /* literal */
                     opt[last_match_pos+addLit].off = 0;
                     opt[last_match_pos+addLit].litlen = addLit;
-- 
cgit v0.12


From bd992f12e4be48b4eed48b3898153f2c80bc014b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 15:05:45 -0700
Subject: searching match leading strictly farther does not work

sometimes, it's better to re-use same match but start it later,
in order to get shorter matchlength code
---
 lib/lz4opt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 25ceaad..1e696f9 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -318,7 +318,7 @@ static int LZ4HC_compress_optimal (
             DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur + 1, matches, fullUpdate);   /* only works if last_match_pos is really the last match pos */
+            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur - 1, matches, fullUpdate);   /* only test matches of a minimum length */
             if (!nb_matches) continue;
 
             if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
-- 
cgit v0.12


From 4b8188580084112f01f0d0913ac022162fd71f25 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 15:37:18 -0700
Subject: partial search, while preserving compression ratio

tag interesting places
---
 lib/lz4hc.c  |  4 ++--
 lib/lz4opt.h | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 884f5d7..44e0b0a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -363,14 +363,14 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
     static const BYTE* start = NULL;
     static U32 totalCost = 0;
-    U32 const pos = (U32)(*anchor - start);
+    U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);
     U32 const ll = (U32)(*ip - *anchor);
     U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
     //g_debuglog_enable = (pos >= 112705) & (pos <= 112760);
-    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u / %u",
+    DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
                 cost, totalCost);
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 1e696f9..37cc73a 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -46,6 +46,7 @@ typedef struct {
     int off;
     int mlen;
     int litlen;
+    int toSearch;
 } LZ4HC_optimal_t;
 
 
@@ -278,6 +279,7 @@ static int LZ4HC_compress_optimal (
                 opt[rPos].off = 0;
                 opt[rPos].litlen = llen + rPos;
                 opt[rPos].price = cost;
+                opt[rPos].toSearch = 1;
                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
                             rPos, cost, opt[rPos].litlen);
         }   }
@@ -294,16 +296,21 @@ static int LZ4HC_compress_optimal (
                     opt[mlen].off = offset;
                     opt[mlen].litlen = llen;
                     opt[mlen].price = cost;
+                    opt[mlen].toSearch = (((mlen - 18) % 255) == 0);
                     DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
                                 mlen, cost, mlen);
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
+        opt[last_match_pos-2].toSearch = 1;
+        opt[last_match_pos-1].toSearch = 1;
+        opt[last_match_pos].toSearch = 1;
         {   int addLit;
             for (addLit = 1; addLit <= 3; addLit ++) {
                 opt[last_match_pos+addLit].mlen = 1; /* literal */
                 opt[last_match_pos+addLit].off = 0;
                 opt[last_match_pos+addLit].litlen = addLit;
                 opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                opt[last_match_pos+addLit].toSearch = 1;
                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
                             last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
         }   }
@@ -314,6 +321,7 @@ static int LZ4HC_compress_optimal (
             int nb_matches;
 
             if (curPtr >= mflimit) break;
+            if (opt[cur].toSearch == 0) continue;
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
@@ -341,6 +349,7 @@ static int LZ4HC_compress_optimal (
                         opt[pos].off = 0;
                         opt[pos].litlen = baseLitlen+litlen;
                         opt[pos].price = price;
+                        opt[pos].toSearch = 1;
                         DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
                                     pos, price, opt[pos].litlen);
             }   }   }
@@ -380,14 +389,19 @@ static int LZ4HC_compress_optimal (
                             opt[pos].off = offset;
                             opt[pos].litlen = ll;
                             opt[pos].price = price;
+                            opt[pos].toSearch = (((ml-18) % 255) == 0);
             }   }   }   }
             /* complete following positions with literals */
+            opt[last_match_pos-2].toSearch = 1;
+            opt[last_match_pos-1].toSearch = 1;
+            opt[last_match_pos].toSearch = 1;
             {   int addLit;
                 for (addLit = 1; addLit <= 3; addLit ++) {
                     opt[last_match_pos+addLit].mlen = 1; /* literal */
                     opt[last_match_pos+addLit].off = 0;
                     opt[last_match_pos+addLit].litlen = addLit;
                     opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
+                    opt[last_match_pos+addLit].toSearch = 1;
                     DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
             }   }
         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
-- 
cgit v0.12


From e06cb03c11ee8821a10fd8496f029c64192c9bc8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 16:25:10 -0700
Subject: small adaptations for intermediate level 11

---
 lib/lz4hc.c  |  2 +-
 lib/lz4opt.h | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 44e0b0a..941cda0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -644,7 +644,7 @@ static int LZ4HC_getSearchNum(int compressionLevel)
 {
     switch (compressionLevel) {
         default: return 0; /* unused */
-        case 11: return 128;
+        case 11: return 256;
         case 12: return 1<<13;
     }
 }
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 37cc73a..c9fab04 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -301,8 +301,8 @@ static int LZ4HC_compress_optimal (
                                 mlen, cost, mlen);
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
-        opt[last_match_pos-2].toSearch = 1;
-        opt[last_match_pos-1].toSearch = 1;
+        if (fullUpdate) opt[last_match_pos-2].toSearch = 1;   /* 1 byte on calgary */
+        if (fullUpdate) opt[last_match_pos-1].toSearch = 1;   /* 1 byte on calgary */
         opt[last_match_pos].toSearch = 1;
         {   int addLit;
             for (addLit = 1; addLit <= 3; addLit ++) {
@@ -349,7 +349,6 @@ static int LZ4HC_compress_optimal (
                         opt[pos].off = 0;
                         opt[pos].litlen = baseLitlen+litlen;
                         opt[pos].price = price;
-                        opt[pos].toSearch = 1;
                         DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)",
                                     pos, price, opt[pos].litlen);
             }   }   }
@@ -392,8 +391,8 @@ static int LZ4HC_compress_optimal (
                             opt[pos].toSearch = (((ml-18) % 255) == 0);
             }   }   }   }
             /* complete following positions with literals */
-            opt[last_match_pos-2].toSearch = 1;
-            opt[last_match_pos-1].toSearch = 1;
+            if (fullUpdate) opt[last_match_pos-2].toSearch = 1;   /* 2 bytes on enwik7 */
+            if (fullUpdate) opt[last_match_pos-1].toSearch = 1;   /* 53 bytes on enwik7, 13 bytes on calgary */
             opt[last_match_pos].toSearch = 1;
             {   int addLit;
                 for (addLit = 1; addLit <= 3; addLit ++) {
@@ -439,7 +438,7 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
                 if (ml == 1) { ip++; rPos++; continue; }  /* literal; note: can end up with several literals, in which case, skip them */
                 rPos += ml;
                 assert(ml >= MINMATCH);
-                assert((offset >= 1) && (offset <=65535));
+                assert((offset >= 1) && (offset <= MAX_DISTANCE));
                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
                     return 0;  /* error */
         }   }
-- 
cgit v0.12


From a1c5343d89cb854462d8458c609a4f61a6aeb808 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 18:54:18 -0700
Subject: more generic skip formula

improving speed
---
 lib/lz4hc.c  |  2 +-
 lib/lz4opt.h | 17 ++++-------------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 941cda0..2d8671d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -369,7 +369,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = *anchor;  /* only works for single segment */
-    //g_debuglog_enable = (pos >= 112705) & (pos <= 112760);
+    //g_debuglog_enable = (pos >= 2228) & (pos <= 2262);
     DEBUGLOG(2, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
                 pos,
                 (U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index c9fab04..0463cb1 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -46,7 +46,6 @@ typedef struct {
     int off;
     int mlen;
     int litlen;
-    int toSearch;
 } LZ4HC_optimal_t;
 
 
@@ -244,6 +243,7 @@ static int LZ4HC_compress_optimal (
     const BYTE* const matchlimit = iend - LASTLITERALS;
     BYTE* op = (BYTE*) dst;
     BYTE* const oend = op + dstCapacity;
+    int const front = fullUpdate ? 2 : 1;
 
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
@@ -279,7 +279,6 @@ static int LZ4HC_compress_optimal (
                 opt[rPos].off = 0;
                 opt[rPos].litlen = llen + rPos;
                 opt[rPos].price = cost;
-                opt[rPos].toSearch = 1;
                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
                             rPos, cost, opt[rPos].litlen);
         }   }
@@ -296,21 +295,16 @@ static int LZ4HC_compress_optimal (
                     opt[mlen].off = offset;
                     opt[mlen].litlen = llen;
                     opt[mlen].price = cost;
-                    opt[mlen].toSearch = (((mlen - 18) % 255) == 0);
                     DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
                                 mlen, cost, mlen);
         }   }   }
         last_match_pos = matches[nb_matches_initial-1].len;
-        if (fullUpdate) opt[last_match_pos-2].toSearch = 1;   /* 1 byte on calgary */
-        if (fullUpdate) opt[last_match_pos-1].toSearch = 1;   /* 1 byte on calgary */
-        opt[last_match_pos].toSearch = 1;
         {   int addLit;
             for (addLit = 1; addLit <= 3; addLit ++) {
                 opt[last_match_pos+addLit].mlen = 1; /* literal */
                 opt[last_match_pos+addLit].off = 0;
                 opt[last_match_pos+addLit].litlen = addLit;
                 opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                opt[last_match_pos+addLit].toSearch = 1;
                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
                             last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
         }   }
@@ -321,7 +315,9 @@ static int LZ4HC_compress_optimal (
             int nb_matches;
 
             if (curPtr >= mflimit) break;
-            if (opt[cur].toSearch == 0) continue;
+            DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
+                    cur, opt[cur].price, opt[cur+1].price, cur+1);
+            if (opt[cur+front].price <= opt[cur].price) continue;
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
@@ -388,19 +384,14 @@ static int LZ4HC_compress_optimal (
                             opt[pos].off = offset;
                             opt[pos].litlen = ll;
                             opt[pos].price = price;
-                            opt[pos].toSearch = (((ml-18) % 255) == 0);
             }   }   }   }
             /* complete following positions with literals */
-            if (fullUpdate) opt[last_match_pos-2].toSearch = 1;   /* 2 bytes on enwik7 */
-            if (fullUpdate) opt[last_match_pos-1].toSearch = 1;   /* 53 bytes on enwik7, 13 bytes on calgary */
-            opt[last_match_pos].toSearch = 1;
             {   int addLit;
                 for (addLit = 1; addLit <= 3; addLit ++) {
                     opt[last_match_pos+addLit].mlen = 1; /* literal */
                     opt[last_match_pos+addLit].off = 0;
                     opt[last_match_pos+addLit].litlen = addLit;
                     opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit);
-                    opt[last_match_pos+addLit].toSearch = 1;
                     DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit);
             }   }
         }  /* for (cur = 1; cur <= last_match_pos; cur++) */
-- 
cgit v0.12


From 05d78eb817fc3eea13bd89377e771bbe19bb02c3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Nov 2017 19:50:08 -0700
Subject: new level 11 uses 512 attempts

---
 lib/lz4hc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 2d8671d..a7238c7 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -644,7 +644,7 @@ static int LZ4HC_getSearchNum(int compressionLevel)
 {
     switch (compressionLevel) {
         default: return 0; /* unused */
-        case 11: return 256;
+        case 11: return 512;
         case 12: return 1<<13;
     }
 }
-- 
cgit v0.12


From 890c0553d08ad6a6a4f2801523667413b5477512 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 00:15:52 -0700
Subject: optimized skip strategy for level 12

---
 lib/lz4opt.h | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 0463cb1..edcfc10 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -243,7 +243,6 @@ static int LZ4HC_compress_optimal (
     const BYTE* const matchlimit = iend - LASTLITERALS;
     BYTE* op = (BYTE*) dst;
     BYTE* const oend = op + dstCapacity;
-    int const front = fullUpdate ? 2 : 1;
 
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
@@ -317,12 +316,16 @@ static int LZ4HC_compress_optimal (
             if (curPtr >= mflimit) break;
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
                     cur, opt[cur].price, opt[cur+1].price, cur+1);
-            if (opt[cur+front].price <= opt[cur].price) continue;
+            if (fullUpdate) {
+                if ((opt[cur+1].price <= opt[cur].price) && (opt[cur+4].price < opt[cur].price+3)) continue;
+            } else {
+                if (opt[cur+1].price <= opt[cur].price) continue;
+            }
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur - 1, matches, fullUpdate);   /* only test matches of a minimum length */
+            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches, fullUpdate);   /* only test matches of a minimum length */
             if (!nb_matches) continue;
 
             if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
-- 
cgit v0.12


From 82c1aed41927d0d3400ab91f975c2e163ecdcc1d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 00:59:05 -0700
Subject: improved level 11 speed

---
 lib/lz4opt.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index edcfc10..c754865 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -324,8 +324,10 @@ static int LZ4HC_compress_optimal (
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
-            //nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches, fullUpdate);   /* only test matches of a minimum length */
+            if (fullUpdate)
+                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+            else
+                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches, fullUpdate);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
             if (!nb_matches) continue;
 
             if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
-- 
cgit v0.12


From 81667a1e966469de0f307bfda141ccb6e89a3115 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 01:18:12 -0700
Subject: removed code and reference to binary tree match finder

reduced size of LZ4HC state
---
 lib/lz4hc.c  |  10 +----
 lib/lz4hc.h  |   4 +-
 lib/lz4opt.h | 124 +----------------------------------------------------------
 3 files changed, 6 insertions(+), 132 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a7238c7..c1f8da6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -761,10 +761,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
     }
     LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
     ctxPtr->end = (const BYTE*)dictionary + dictSize;
-    if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
-        LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
-    else
-        if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
+    if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
     return dictSize;
 }
 
@@ -773,10 +770,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
 
 static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
 {
-    if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
-        LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
-    else
-        if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
 
     /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
     ctxPtr->lowLimit  = ctxPtr->dictLimit;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index a9cefbb..191ab0c 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -129,7 +129,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
  * They are exposed to allow static allocation of `LZ4_streamHC_t`.
  * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
  **************************************/
-#define LZ4HC_DICTIONARY_LOGSIZE 17   /* due to btree, hc would only need 16 */
+#define LZ4HC_DICTIONARY_LOGSIZE 16
 #define LZ4HC_MAXD (1<= MINMATCH */
 LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 {
-    int price = 2 + 1; /* 16-bit offset + token */
+    int price = 1 + 2 ; /* token + 16-bit offset */
 
     price += LZ4HC_literalsPrice(litlen);
 
@@ -74,126 +74,8 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 
 
 /*-*************************************
-*  Binary Tree search
+*  Match finder
 ***************************************/
-LZ4_FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
-    LZ4HC_CCtx_internal* ctx,
-    const BYTE* const ip,
-    const BYTE* const iHighLimit,
-    size_t best_mlen,
-    LZ4HC_match_t* matches,
-    int* matchNum)
-{
-    U16* const chainTable = ctx->chainTable;
-    U32* const HashTable = ctx->hashTable;
-    const BYTE* const base = ctx->base;
-    const U32 dictLimit = ctx->dictLimit;
-    const U32 current = (U32)(ip - base);
-    const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1);
-    const BYTE* const dictBase = ctx->dictBase;
-    const BYTE* match;
-    int nbAttempts = ctx->searchNum;
-    int mnum = 0;
-    U16 *ptr0, *ptr1, delta0, delta1;
-    U32 matchIndex;
-    size_t matchLength = 0;
-    U32* HashPos;
-
-    if (ip + MINMATCH > iHighLimit) return 1;
-
-    /* HC4 match finder */
-    HashPos = &HashTable[LZ4HC_hashPtr(ip)];
-    matchIndex = *HashPos;
-    *HashPos = current;
-
-    ptr0 = &DELTANEXTMAXD(current*2+1);
-    ptr1 = &DELTANEXTMAXD(current*2);
-    delta0 = delta1 = (U16)(current - matchIndex);
-
-    while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) {
-        nbAttempts--;
-        if (matchIndex >= dictLimit) {
-            match = base + matchIndex;
-            matchLength = LZ4_count(ip, match, iHighLimit);
-        } else {
-            const BYTE* vLimit = ip + (dictLimit - matchIndex);
-            match = dictBase + matchIndex;
-            if (vLimit > iHighLimit) vLimit = iHighLimit;
-            matchLength = LZ4_count(ip, match, vLimit);
-            if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
-                matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit);
-            if (matchIndex+matchLength >= dictLimit)
-                match = base + matchIndex;   /* to prepare for next usage of match[matchLength] */
-        }
-
-        if (matchLength > best_mlen) {
-            best_mlen = matchLength;
-            if (matches) {
-                if (matchIndex >= dictLimit)
-                    matches[mnum].off = (int)(ip - match);
-                else
-                    matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */
-                matches[mnum].len = (int)matchLength;
-                mnum++;
-            }
-            if (best_mlen > LZ4_OPT_NUM) break;
-        }
-
-        if (ip+matchLength >= iHighLimit)   /* equal : no way to know if inf or sup */
-            break;   /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
-
-        DEBUGLOG(6, "ip   :%016llX", (U64)ip);
-        DEBUGLOG(6, "match:%016llX", (U64)match);
-        if (*(ip+matchLength) < *(match+matchLength)) {
-            *ptr0 = delta0;
-            ptr0 = &DELTANEXTMAXD(matchIndex*2);
-            if (*ptr0 == (U16)-1) break;
-            delta0 = *ptr0;
-            delta1 += delta0;
-            matchIndex -= delta0;
-        } else {
-            *ptr1 = delta1;
-            ptr1 = &DELTANEXTMAXD(matchIndex*2+1);
-            if (*ptr1 == (U16)-1) break;
-            delta1 = *ptr1;
-            delta0 += delta1;
-            matchIndex -= delta1;
-        }
-    }
-
-    *ptr0 = (U16)-1;
-    *ptr1 = (U16)-1;
-    if (matchNum) *matchNum = mnum;
-  /*  if (best_mlen > 8) return best_mlen-8; */
-    if (!matchNum) return 1;
-    return 1;
-}
-
-
-LZ4_FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit)
-{
-    const BYTE* const base = ctx->base;
-    const U32 target = (U32)(ip - base);
-    U32 idx = ctx->nextToUpdate;
-    while(idx < target)
-        idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
-}
-
-
-/** Tree updater, providing best match */
-LZ4_FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
-                        LZ4HC_CCtx_internal* ctx,
-                        const BYTE* const ip, const BYTE* const iHighLimit,
-                        size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
-{
-    int mnum = 0;
-    if (ip < ctx->base + ctx->nextToUpdate) return 0;   /* skipped area */
-    if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit);
-    best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum);
-    ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen);
-    return mnum;
-}
-
 
 LZ4_FORCE_INLINE
 int LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,   /* Index table will be updated */
@@ -257,7 +139,6 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        //int const nb_matches_initial = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         int const nb_matches_initial = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
         if (!nb_matches_initial) { ip++; continue; }
 
@@ -323,7 +204,6 @@ static int LZ4HC_compress_optimal (
             }
 
             DEBUGLOG(7, "search at rPos:%u", cur);
-            //nb_matches = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             if (fullUpdate)
                 nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
             else
-- 
cgit v0.12


From 320e1d51ac6fd456107c18ed9d6b2c71d7ffa49f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 01:20:30 -0700
Subject: removed useless parameter from hash chain matchfinder

used to be present for compatibility with binary tree matchfinder
---
 lib/lz4opt.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 841e00c..6560ec5 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -91,7 +91,7 @@ int LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,   /* Index table will
 LZ4_FORCE_INLINE int LZ4HC_HashChain_GetAllMatches (
                         LZ4HC_CCtx_internal* ctx,
                         const BYTE* const ip, const BYTE* const iHighLimit,
-                        size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
+                        size_t best_mlen, LZ4HC_match_t* matches)
 {
     const BYTE* matchPtr = NULL;
     int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, ctx->searchNum);
@@ -99,7 +99,6 @@ LZ4_FORCE_INLINE int LZ4HC_HashChain_GetAllMatches (
     assert(matches != NULL);
     matches[0].len = matchLength;
     matches[0].off = (int)(ip-matchPtr);
-    (void)fullUpdate;
     return 1;
 }
 
@@ -139,7 +138,7 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        int const nb_matches_initial = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
+        int const nb_matches_initial = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches);
         if (!nb_matches_initial) { ip++; continue; }
 
         if ((size_t)matches[nb_matches_initial-1].len > sufficient_len) {
@@ -205,9 +204,9 @@ static int LZ4HC_compress_optimal (
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             if (fullUpdate)
-                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate);
+                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches);
             else
-                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches, fullUpdate);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
+                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
             if (!nb_matches) continue;
 
             if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
-- 
cgit v0.12


From 3b222d2d963fe6144498a3cb306943566ddb6922 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 01:37:43 -0700
Subject: removes matches[] table

saves stack space
clearer match finder interface (no more table to fill)
---
 lib/lz4opt.h | 140 ++++++++++++++++++++++++++++-------------------------------
 1 file changed, 67 insertions(+), 73 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 6560ec5..a1627f1 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -88,18 +88,18 @@ int LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,   /* Index table will
     return LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, longest, matchpos, &uselessPtr, maxNbAttempts);
 }
 
-LZ4_FORCE_INLINE int LZ4HC_HashChain_GetAllMatches (
-                        LZ4HC_CCtx_internal* ctx,
+LZ4_FORCE_INLINE
+LZ4HC_match_t LZ4HC_HashChain_GetAllMatches (LZ4HC_CCtx_internal* const ctx,
                         const BYTE* const ip, const BYTE* const iHighLimit,
-                        size_t best_mlen, LZ4HC_match_t* matches)
+                        size_t best_mlen)
 {
+    LZ4HC_match_t match = {0 , 0};
     const BYTE* matchPtr = NULL;
     int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, ctx->searchNum);
-    if ((size_t)matchLength <= best_mlen) return 0;
-    assert(matches != NULL);
-    matches[0].len = matchLength;
-    matches[0].off = (int)(ip-matchPtr);
-    return 1;
+    if ((size_t)matchLength <= best_mlen) return match;
+    match.len = matchLength;
+    match.off = (int)(ip-matchPtr);
+    return match;
 }
 
 
@@ -115,7 +115,6 @@ static int LZ4HC_compress_optimal (
     )
 {
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + 3];   /* this uses a bit too much stack memory to my taste ... */
-    LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -138,13 +137,13 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        int const nb_matches_initial = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches);
-        if (!nb_matches_initial) { ip++; continue; }
+        LZ4HC_match_t const firstMatch = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1);
+        if (firstMatch.len==0) { ip++; continue; }
 
-        if ((size_t)matches[nb_matches_initial-1].len > sufficient_len) {
+        if ((size_t)firstMatch.len > sufficient_len) {
             /* good enough solution : immediate encoding */
-            int const firstML = matches[nb_matches_initial-1].len;
-            const BYTE* const matchPos = ip - matches[nb_matches_initial-1].off;
+            int const firstML = firstMatch.len;
+            const BYTE* const matchPos = ip - firstMatch.off;
             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
                 return 0;  /* error */
             continue;
@@ -162,22 +161,20 @@ static int LZ4HC_compress_optimal (
                             rPos, cost, opt[rPos].litlen);
         }   }
         /* set prices using matches found for rPos = 0 */
-        {   int matchNb;
-            for (matchNb = 0; matchNb < nb_matches_initial; matchNb++) {
-                int mlen = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
-                int const matchML = matches[matchNb].len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
-                int const offset = matches[matchNb].off;
-                assert(matchML < LZ4_OPT_NUM);
-                for ( ; mlen <= matchML ; mlen++) {
-                    int const cost = LZ4HC_sequencePrice(llen, mlen);
-                    opt[mlen].mlen = mlen;
-                    opt[mlen].off = offset;
-                    opt[mlen].litlen = llen;
-                    opt[mlen].price = cost;
-                    DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
-                                mlen, cost, mlen);
-        }   }   }
-        last_match_pos = matches[nb_matches_initial-1].len;
+        {   int mlen = MINMATCH;
+            int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
+            int const offset = firstMatch.off;
+            assert(matchML < LZ4_OPT_NUM);
+            for ( ; mlen <= matchML ; mlen++) {
+                int const cost = LZ4HC_sequencePrice(llen, mlen);
+                opt[mlen].mlen = mlen;
+                opt[mlen].off = offset;
+                opt[mlen].litlen = llen;
+                opt[mlen].price = cost;
+                DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup",
+                            mlen, cost, mlen);
+        }   }
+        last_match_pos = firstMatch.len;
         {   int addLit;
             for (addLit = 1; addLit <= 3; addLit ++) {
                 opt[last_match_pos+addLit].mlen = 1; /* literal */
@@ -191,7 +188,7 @@ static int LZ4HC_compress_optimal (
         /* check further positions */
         for (cur = 1; cur < last_match_pos; cur++) {
             const BYTE* const curPtr = ip + cur;
-            int nb_matches;
+            LZ4HC_match_t newMatch;
 
             if (curPtr >= mflimit) break;
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
@@ -204,16 +201,16 @@ static int LZ4HC_compress_optimal (
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             if (fullUpdate)
-                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches);
+                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1);
             else
-                nb_matches = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, matches);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
-            if (!nb_matches) continue;
+                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
+            if (!newMatch.len) continue;
 
-            if ( ((size_t)matches[nb_matches-1].len > sufficient_len)
-              || (matches[nb_matches-1].len + cur >= LZ4_OPT_NUM) ) {
+            if ( ((size_t)newMatch.len > sufficient_len)
+              || (newMatch.len + cur >= LZ4_OPT_NUM) ) {
                 /* immediate encoding */
-                best_mlen = matches[nb_matches-1].len;
-                best_off = matches[nb_matches-1].off;
+                best_mlen = newMatch.len;
+                best_off = newMatch.off;
                 last_match_pos = cur + 1;
                 goto encode;
             }
@@ -234,41 +231,38 @@ static int LZ4HC_compress_optimal (
             }   }   }
 
             /* set prices using matches at position = cur */
-            {   int matchNb;
-                assert(cur + matches[nb_matches-1].len < LZ4_OPT_NUM);
-                for (matchNb = 0; matchNb < nb_matches; matchNb++) {
-                    int const matchML = matches[matchNb].len;
-                    int ml = (matchNb>0) ? matches[matchNb-1].len+1 : MINMATCH;
-
-                    for ( ; ml <= matchML ; ml++) {
-                        int const pos = cur + ml;
-                        int const offset = matches[matchNb].off;
-                        int price;
-                        int ll;
-                        DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
-                                    pos, last_match_pos);
-                        if (opt[cur].mlen == 1) {
-                            ll = opt[cur].litlen;
-                            price = ((cur > ll) ? opt[cur - ll].price : 0)
-                                  + LZ4HC_sequencePrice(ll, ml);
-                        } else {
-                            ll = 0;
-                            price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
-                        }
-
-                        if (pos > last_match_pos+3 || price <= opt[pos].price) {
-                            DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
-                                        pos, price, ml);
-                            assert(pos < LZ4_OPT_NUM);
-                            if ( (matchNb == nb_matches-1)  /* last match */
-                              && (ml == matchML)  /* last post of last match */
-                              && (last_match_pos < pos) )
-                                last_match_pos = pos;
-                            opt[pos].mlen = ml;
-                            opt[pos].off = offset;
-                            opt[pos].litlen = ll;
-                            opt[pos].price = price;
-            }   }   }   }
+            {   int const matchML = newMatch.len;
+                int ml = MINMATCH;
+                
+                assert(cur + newMatch.len < LZ4_OPT_NUM);
+                for ( ; ml <= matchML ; ml++) {
+                    int const pos = cur + ml;
+                    int const offset = newMatch.off;
+                    int price;
+                    int ll;
+                    DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)",
+                                pos, last_match_pos);
+                    if (opt[cur].mlen == 1) {
+                        ll = opt[cur].litlen;
+                        price = ((cur > ll) ? opt[cur - ll].price : 0)
+                              + LZ4HC_sequencePrice(ll, ml);
+                    } else {
+                        ll = 0;
+                        price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
+                    }
+
+                    if (pos > last_match_pos+3 || price <= opt[pos].price) {
+                        DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
+                                    pos, price, ml);
+                        assert(pos < LZ4_OPT_NUM);
+                        if ( (ml == matchML)  /* last post of last match */
+                          && (last_match_pos < pos) )
+                            last_match_pos = pos;
+                        opt[pos].mlen = ml;
+                        opt[pos].off = offset;
+                        opt[pos].litlen = ll;
+                        opt[pos].price = price;
+            }   }   }
             /* complete following positions with literals */
             {   int addLit;
                 for (addLit = 1; addLit <= 3; addLit ++) {
-- 
cgit v0.12


From e2eca62046e500a95ab34e913e939aa68acb2cd4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 02:01:20 -0700
Subject: LZ4_compress_HC_continue_destSize() now compatible with optimal
 parser

levels 11+
---
 lib/lz4hc.c  | 11 ++++++-----
 lib/lz4hc.h  |  4 ++--
 lib/lz4opt.h | 10 +++++-----
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index c1f8da6..643fbb2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -642,8 +642,11 @@ _dest_overflow:
 
 static int LZ4HC_getSearchNum(int compressionLevel)
 {
+    assert(compressionLevel >= 1);
+    assert(compressionLevel <= LZ4HC_CLEVEL_MAX);
     switch (compressionLevel) {
-        default: return 0; /* unused */
+        default: return 1 << (compressionLevel-1);
+        case 10: return 1 << 12;
         case 11: return 512;
         case 12: return 1<<13;
     }
@@ -709,8 +712,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 }
 
 /* LZ4_compress_HC_destSize() :
- * currently, only compatible with Hash Chain implementation,
- * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/
+ * only compatible with Hash Chain match finder */
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
@@ -744,6 +746,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
+    /* note : 1-10 / 11-12 separation might no longer be necessary since optimal parser uses hash chain too */
     int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel;
     int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN;
     int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX;
@@ -824,8 +827,6 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src,
 
 int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize)
 {
-    LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
-    if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src);   /* not compatible with btopt implementation */
     return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize);
 }
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 191ab0c..4fbe91e 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -266,8 +266,8 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
                             int* srcSizePtr, int targetDstSize);
 
 /*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
- *  It's possible to change compression level after LZ4_resetStreamHC(), between 2 invocations of LZ4_compress_HC_continue*(),
- *  but that requires to stay in the same mode (aka 1-10 or 11-12).
+ *  It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*(),
+ *  but it requires to stay in the same mode (aka 1-10 or 11-12).
  *  This function ensures this condition.
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index a1627f1..18bcd31 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -160,7 +160,7 @@ static int LZ4HC_compress_optimal (
                 DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup",
                             rPos, cost, opt[rPos].litlen);
         }   }
-        /* set prices using matches found for rPos = 0 */
+        /* set prices using initial match */
         {   int mlen = MINMATCH;
             int const matchML = firstMatch.len;   /* necessarily < sufficient_len < LZ4_OPT_NUM */
             int const offset = firstMatch.off;
@@ -215,7 +215,7 @@ static int LZ4HC_compress_optimal (
                 goto encode;
             }
 
-            /* before first match : set price with literals at beginning */
+            /* before match : set price with literals at beginning */
             {   int const baseLitlen = opt[cur].litlen;
                 int litlen;
                 for (litlen = 1; litlen < MINMATCH; litlen++) {
@@ -230,10 +230,10 @@ static int LZ4HC_compress_optimal (
                                     pos, price, opt[pos].litlen);
             }   }   }
 
-            /* set prices using matches at position = cur */
+            /* set prices using match at position = cur */
             {   int const matchML = newMatch.len;
                 int ml = MINMATCH;
-                
+
                 assert(cur + newMatch.len < LZ4_OPT_NUM);
                 for ( ; ml <= matchML ; ml++) {
                     int const pos = cur + ml;
@@ -255,7 +255,7 @@ static int LZ4HC_compress_optimal (
                         DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
                                     pos, price, ml);
                         assert(pos < LZ4_OPT_NUM);
-                        if ( (ml == matchML)  /* last post of last match */
+                        if ( (ml == matchML)  /* last pos of last match */
                           && (last_match_pos < pos) )
                             last_match_pos = pos;
                         opt[pos].mlen = ml;
-- 
cgit v0.12


From c9bbad53ffda4a4f4595e3a2b6e3994710ee2324 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 10:30:52 -0700
Subject: removed ctx->searchNum

nbSearches now transmitted directly as function parameter
easier to track and debug
---
 lib/lz4hc.c  | 20 +++-----------------
 lib/lz4hc.h  |  4 +---
 lib/lz4opt.h | 14 ++++++++------
 3 files changed, 12 insertions(+), 26 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 643fbb2..a7577c5 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -640,17 +640,6 @@ _dest_overflow:
     return 0;
 }
 
-static int LZ4HC_getSearchNum(int compressionLevel)
-{
-    assert(compressionLevel >= 1);
-    assert(compressionLevel <= LZ4HC_CLEVEL_MAX);
-    switch (compressionLevel) {
-        default: return 1 << (compressionLevel-1);
-        case 10: return 1 << 12;
-        case 11: return 512;
-        case 12: return 1<<13;
-    }
-}
 
 static int LZ4HC_compress_generic (
     LZ4HC_CCtx_internal* const ctx,
@@ -667,16 +656,14 @@ static int LZ4HC_compress_generic (
         if (limit == limitedDestSize) cLevel = 10;
         switch (cLevel) {
             case 10:
-                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << 12, limit);
+                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1<<12, limit);
             case 11:
-                ctx->searchNum = LZ4HC_getSearchNum(cLevel);
-                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 128, 0);
+                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 512, 128, 0);
             default:
                 cLevel = 12;
                 /* fall-through */
             case 12:
-                ctx->searchNum = LZ4HC_getSearchNum(cLevel);
-                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, LZ4_OPT_NUM, 1);
+                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 1<<13, LZ4_OPT_NUM, 1);
         }
     }
     return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit);  /* levels 1-9 */
@@ -741,7 +728,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
     if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;  /* cap compression level */
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
-    LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
 }
 
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 4fbe91e..13a0179 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -152,8 +152,7 @@ typedef struct
     uint32_t   dictLimit;       /* below that point, need extDict */
     uint32_t   lowLimit;        /* below that point, no more dict */
     uint32_t   nextToUpdate;    /* index from which to continue dictionary update */
-    uint32_t   searchNum;       /* only for optimal parser */
-    uint32_t   compressionLevel;
+    int        compressionLevel;
 } LZ4HC_CCtx_internal;
 
 #else
@@ -169,7 +168,6 @@ typedef struct
     unsigned int   dictLimit;        /* below that point, need extDict */
     unsigned int   lowLimit;         /* below that point, no more dict */
     unsigned int   nextToUpdate;     /* index from which to continue dictionary update */
-    unsigned int   searchNum;        /* only for optimal parser */
     int            compressionLevel;
 } LZ4HC_CCtx_internal;
 
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 18bcd31..7aec7dd 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -91,11 +91,11 @@ int LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,   /* Index table will
 LZ4_FORCE_INLINE
 LZ4HC_match_t LZ4HC_HashChain_GetAllMatches (LZ4HC_CCtx_internal* const ctx,
                         const BYTE* const ip, const BYTE* const iHighLimit,
-                        size_t best_mlen)
+                        size_t best_mlen, int nbSearches)
 {
     LZ4HC_match_t match = {0 , 0};
     const BYTE* matchPtr = NULL;
-    int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, ctx->searchNum);
+    int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, nbSearches);
     if ((size_t)matchLength <= best_mlen) return match;
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
@@ -110,8 +110,9 @@ static int LZ4HC_compress_optimal (
     int inputSize,
     int dstCapacity,
     limitedOutput_directive limit,
+    int const nbSearches,
     size_t sufficient_len,
-    const int fullUpdate
+    int const fullUpdate
     )
 {
     LZ4HC_optimal_t opt[LZ4_OPT_NUM + 3];   /* this uses a bit too much stack memory to my taste ... */
@@ -137,7 +138,7 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        LZ4HC_match_t const firstMatch = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1);
+        LZ4HC_match_t const firstMatch = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
         if (firstMatch.len==0) { ip++; continue; }
 
         if ((size_t)firstMatch.len > sufficient_len) {
@@ -201,9 +202,10 @@ static int LZ4HC_compress_optimal (
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             if (fullUpdate)
-                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1);
+                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
             else
-                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur);   /* only test matches of a minimum length; slightly faster, but misses a few bytes */
+                /* only test matches of minimum length; slightly faster, but misses a few bytes */
+                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
             if (!newMatch.len) continue;
 
             if ( ((size_t)newMatch.len > sufficient_len)
-- 
cgit v0.12


From a1f4a0d98361c6ff95833dedba07a9d5a15cfa5e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 10:48:55 -0700
Subject: moved ctx->end handling from parsers

responsibility better handled one layer above (LZ4HC_compress_generic())
---
 lib/lz4hc.c  | 4 ++--
 lib/lz4opt.h | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index a7577c5..cea83f2 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -457,7 +457,6 @@ static int LZ4HC_compress_hashChain (
     if (limit == limitedDestSize && maxOutputSize < 1) return 0;         /* Impossible to store anything */
     if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;              /* Unsupported input size, too large (or negative) */
 
-    ctx->end += inputSize;
     if (limit == limitedDestSize) oend -= LASTLITERALS;                  /* Hack for support limitations LZ4 decompressor */
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
@@ -651,7 +650,8 @@ static int LZ4HC_compress_generic (
     limitedOutput_directive limit
     )
 {
-    if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe to reconsider */
+    ctx->end += *srcSizePtr;
+    if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe something to review */
     if (cLevel > 9) {
         if (limit == limitedDestSize) cLevel = 10;
         switch (cLevel) {
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 7aec7dd..df8ecd6 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -128,7 +128,6 @@ static int LZ4HC_compress_optimal (
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
-    ctx->end += inputSize;
     ip++;
 
     /* Main Loop */
-- 
cgit v0.12


From 1025546347d75ec94f584294a36132527a45d46c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 11:28:28 -0700
Subject: unified HC levels

LZ4_setCompressionLevel() can be users accross the whole range of HC levels
No more transition issue between Optimal and HC modes
---
 lib/lz4hc.c    | 12 ++++--------
 lib/lz4hc.h    |  3 +--
 tests/fuzzer.c | 12 ++++++------
 3 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index cea83f2..042e034 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -49,6 +49,7 @@
 
 
 /*===    Dependency    ===*/
+#define LZ4_HC_STATIC_LINKING_ONLY
 #include "lz4hc.h"
 
 
@@ -726,18 +727,13 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
     LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET);   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
     LZ4_streamHCPtr->internal_donotuse.base = NULL;
-    if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;  /* cap compression level */
-    LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
+    LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel);
 }
 
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
 {
-    /* note : 1-10 / 11-12 separation might no longer be necessary since optimal parser uses hash chain too */
-    int const currentCLevel = LZ4_streamHCPtr->internal_donotuse.compressionLevel;
-    int const minCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? 1 : LZ4HC_CLEVEL_OPT_MIN;
-    int const maxCLevel = currentCLevel < LZ4HC_CLEVEL_OPT_MIN ? LZ4HC_CLEVEL_OPT_MIN-1 : LZ4HC_CLEVEL_MAX;
-    compressionLevel = MIN(compressionLevel, minCLevel);
-    compressionLevel = MAX(compressionLevel, maxCLevel);
+    if (compressionLevel < 1) compressionLevel = 1;
+    if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX;
     LZ4_streamHCPtr->internal_donotuse.compressionLevel = compressionLevel;
 }
 
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 13a0179..04153e6 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -265,8 +265,7 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
 
 /*! LZ4_setCompressionLevel() : v1.8.0 (experimental)
  *  It's possible to change compression level between 2 invocations of LZ4_compress_HC_continue*(),
- *  but it requires to stay in the same mode (aka 1-10 or 11-12).
- *  This function ensures this condition.
+ *  though it requires to stay in the same mode (aka fast or HC).
  */
 void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 2e22912..ddd293c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -376,9 +376,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
                 FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
                 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
-                { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
-                  FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
-
+                {   U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
+                    FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
+                }
                 DISPLAYLEVEL(5, " OK \n");
             }
             else
@@ -460,8 +460,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         // Test decoding with output size being 10 bytes too short => must fail
         FUZ_DISPLAYTEST;
-        if (blockSize>10)
-        {
+        if (blockSize>10) {
             decodedBuffer[blockSize-10] = 0;
             ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
             FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
@@ -632,6 +631,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
         LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
         LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
+        LZ4_setCompressionLevel(&LZ4dictHC, compressionLevel-1);
         blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
         FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
 
@@ -657,7 +657,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
 
         /* Compress HC continue destSize */
-            FUZ_DISPLAYTEST;
+        FUZ_DISPLAYTEST;
         {   int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
             int consumedSize = blockSize;
             FUZ_DISPLAYTEST;
-- 
cgit v0.12


From 89821ac7a1e9bdeef33c747321b6deec66cbde1e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 3 Nov 2017 11:49:56 -0700
Subject: minor comment edit

---
 lib/lz4frame.c |  8 ++++----
 lib/lz4hc.h    | 41 +++++++++++++++++++----------------------
 lib/lz4opt.h   | 13 ++++++-------
 3 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 3adbdd9..ebd1089 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -322,7 +322,7 @@ size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
                                const LZ4F_preferences_t* preferencesPtr)
 {
     LZ4F_cctx_t cctxI;
-    LZ4_stream_t lz4ctx;
+    LZ4_stream_t lz4ctx;   /* pretty large on stack */
     LZ4F_preferences_t prefs;
     LZ4F_compressOptions_t options;
     BYTE* const dstStart = (BYTE*) dstBuffer;
@@ -504,15 +504,15 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
     cctxPtr->prefs = *preferencesPtr;
 
     /* Ctx Management */
-    {   U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
-        if (cctxPtr->lz4CtxLevel < tableID) {
+    {   U32 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
+        if (cctxPtr->lz4CtxLevel < ctxTypeID) {
             FREEMEM(cctxPtr->lz4CtxPtr);
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
             else
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
             if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
-            cctxPtr->lz4CtxLevel = tableID;
+            cctxPtr->lz4CtxLevel = ctxTypeID;
     }   }
 
     /* Buffer Management */
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 04153e6..d791062 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -39,7 +39,7 @@ extern "C" {
 #endif
 
 /* --- Dependency --- */
-/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */
+/* note : lz4hc requires lz4.h/lz4.c for compilation */
 #include "lz4.h"   /* stddef, LZ4LIB_API, LZ4_DEPRECATED */
 
 
@@ -54,12 +54,12 @@ extern "C" {
  *  Block Compression
  **************************************/
 /*! LZ4_compress_HC() :
- * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
+ *  Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm.
  * `dst` must be already allocated.
- * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
- * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
- * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work.
- *                      Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
+ *  Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
+ *  Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
+ * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work.
+ *                      Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
  * @return : the number of bytes written into 'dst'
  *           or 0 if compression fails.
  */
@@ -72,12 +72,12 @@ LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dst
 
 
 /*! LZ4_compress_HC_extStateHC() :
- * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
+ *  Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
  * `state` size is provided by LZ4_sizeofStateHC().
- * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
+ *  Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly).
  */
-LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
 LZ4LIB_API int LZ4_sizeofStateHC(void);
+LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
 
 
 /*-************************************
@@ -87,10 +87,10 @@ LZ4LIB_API int LZ4_sizeofStateHC(void);
  typedef union LZ4_streamHC_u LZ4_streamHC_t;   /* incomplete type (defined later) */
 
 /*! LZ4_createStreamHC() and LZ4_freeStreamHC() :
- * These functions create and release memory for LZ4 HC streaming state.
- * Newly created states are automatically initialized.
- * Existing states can be re-used several times, using LZ4_resetStreamHC().
- * These methods are API and ABI stable, they can be used in combination with a DLL.
+ *  These functions create and release memory for LZ4 HC streaming state.
+ *  Newly created states are automatically initialized.
+ *  Existing states can be re-used several times, using LZ4_resetStreamHC().
+ *  These methods are API and ABI stable, they can be used in combination with a DLL.
  */
 LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
 LZ4LIB_API int             LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
@@ -123,12 +123,12 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
 */
 
 
- /*-*************************************
+/*-**************************************************************
  * PRIVATE DEFINITIONS :
  * Do not use these definitions.
  * They are exposed to allow static allocation of `LZ4_streamHC_t`.
  * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
- **************************************/
+ ****************************************************************/
 #define LZ4HC_DICTIONARY_LOGSIZE 16
 #define LZ4HC_MAXD (1<
Date: Fri, 3 Nov 2017 12:33:55 -0700
Subject: fixed minor static analyzer warning

dead assignment
---
 lib/lz4hc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 042e034..0cda77c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -661,7 +661,6 @@ static int LZ4HC_compress_generic (
             case 11:
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 512, 128, 0);
             default:
-                cLevel = 12;
                 /* fall-through */
             case 12:
                 return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 1<<13, LZ4_OPT_NUM, 1);
-- 
cgit v0.12


From cca7618f09b0a98b05a27597f25fcdc68ab908ed Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru 
Date: Sun, 5 Nov 2017 11:48:00 +0100
Subject: When building with a C++ compiler, remove the 'register' keyword to
 silent a warning

For example, with clang:

lz4.c:XXX:36: error: 'register' storage class specifier is deprecated and incompatible with C++17 [-Werror,-Wdeprecated-register]
static unsigned LZ4_NbCommonBytes (register reg_t val)
                                   ^~~~~~~~~
---
 lib/lz4.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 5efcbc0..71acfce 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -85,6 +85,17 @@
 #endif
 
 
+/*
+ * register is ignored when the code built with a C++ compiler
+ * Remove the keyword when built with C++ to silent the warning
+ */
+#ifdef __cplusplus
+#  define REGISTER
+#else
+#  define REGISTER register
+#endif
+
+
 /*-************************************
 *  Dependency
 **************************************/
@@ -330,7 +341,7 @@ static const int LZ4_minLength = (MFLIMIT+1);
 /*-************************************
 *  Common functions
 **************************************/
-static unsigned LZ4_NbCommonBytes (register reg_t val)
+static unsigned LZ4_NbCommonBytes (REGISTER reg_t val)
 {
     if (LZ4_isLittleEndian()) {
         if (sizeof(val)==8) {
-- 
cgit v0.12


From 4fed595dac0a10fb68c77a71370638ad958ccce8 Mon Sep 17 00:00:00 2001
From: Sylvestre Ledru 
Date: Mon, 6 Nov 2017 16:16:02 +0100
Subject: Only ignore with C++17

---
 lib/lz4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 71acfce..64a2e82 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -86,10 +86,10 @@
 
 
 /*
- * register is ignored when the code built with a C++ compiler
- * Remove the keyword when built with C++ to silent the warning
+ * register is ignored when the code built with a C++-17 compiler
+ * Remove the keyword when built with C++-17 to silent the warning
  */
-#ifdef __cplusplus
+#if defined(__cplusplus) &&  __cplusplus > 201402L
 #  define REGISTER
 #else
 #  define REGISTER register
-- 
cgit v0.12


From ce8393e8d7c55e69c73ccff0c37c32f5b0e86b6c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 6 Nov 2017 15:20:08 -0800
Subject: build: minor : `make lz4` doesn't compile liblz4 anymore

since it's not needed.
---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index da485a1..76fc9a8 100644
--- a/Makefile
+++ b/Makefile
@@ -57,12 +57,12 @@ all: allmost manuals
 .PHONY: allmost
 allmost: lib lz4 examples
 
-.PHONY: lib lib-release
-lib lib-release:
+.PHONY: lib lib-release liblz4.a
+lib lib-release liblz4.a:
 	@$(MAKE) -C $(LZ4DIR) $@
 
 .PHONY: lz4 lz4-release
-lz4 : lib
+lz4 : liblz4.a
 lz4-release : lib-release
 lz4 lz4-release :
 	@$(MAKE) -C $(PRGDIR) $@
-- 
cgit v0.12


From d51f0466289d9a021291e736b463cf8de7bd60bd Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 6 Nov 2017 15:42:50 -0800
Subject: 2-stages LZ4_count

separate first branch from the rest of the compare loop
to get dedicated prediction.

measured a 3-4% compression speed improvement.
---
 lib/lz4.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index 64a2e82..ff6496c 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -407,7 +407,15 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi
 {
     const BYTE* const pStart = pIn;
 
-    while (likely(pIn
Date: Mon, 6 Nov 2017 17:29:27 -0800
Subject: added LZ4_FORCEINLINE to counter gcc regression

as recommended by @terrelln
---
 lib/lz4.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index ff6496c..6157285 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -403,7 +403,8 @@ static unsigned LZ4_NbCommonBytes (REGISTER reg_t val)
 }
 
 #define STEPSIZE sizeof(reg_t)
-static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+LZ4_FORCE_INLINE
+unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
 {
     const BYTE* const pStart = pIn;
 
-- 
cgit v0.12


From a004c1fbee4e5a8793f9dfc1deeafea1d59be486 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 10:53:29 -0800
Subject: fixed LZ4HC_countPattern()

- works with byte values other than `0`
- works for any repetitive pattern of length 1, 2 or 4 (but not 3!)
- works for little and big endian systems
- preserve speed of previous implementation
---
 lib/lz4hc.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 0cda77c..fbcec98 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -130,20 +130,34 @@ static int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
     return back;
 }
 
-static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, reg_t pattern)
+/* LZ4HC_countPattern() :
+ * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */
+static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
 {
     const BYTE* const iStart = ip;
+    reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;
 
-    while (likely(ip>= 8;
+        }
+    } else {  /* big endian */
+        U32 bitOffset = (sizeof(pattern)*8) - 8;
+        while (ip < iEnd) {
+            BYTE const byte = (BYTE)(pattern >> bitOffset);
+            if (*ip != byte) break;
+            ip ++; bitOffset -= 8;
+        }
+    }
+
     return (unsigned)(ip - iStart);
 }
 
-- 
cgit v0.12


From 7130bfe5733347d8c84b8f6b8fb34c1397d4a173 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 11:05:48 -0800
Subject: improved LZ4HC_reverseCountPattern() :

works for any repetitive pattern of length 1, 2 or 4 (but not 3!)
works for any endianess
---
 lib/lz4hc.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index fbcec98..1880d53 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -161,17 +161,21 @@ static unsigned LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 c
     return (unsigned)(ip - iStart);
 }
 
+/* LZ4HC_reverseCountPattern() :
+ * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!)
+ * read using natural platform endianess */
 static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern)
 {
     const BYTE* const iStart = ip;
 
-    while (likely(ip>=iLow+4)) {
+    while (likely(ip >= iLow+4)) {
         if (LZ4_read32(ip-4) != pattern) break;
         ip -= 4;
     }
     while (likely(ip>iLow)) {
-        if (ip[-1] != (BYTE)pattern) break;
-        ip--;
+        const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
+        if (ip[-1] != *bytePtr) break;
+        ip--; bytePtr--;
     }
 
     return (unsigned)(iStart - ip);
-- 
cgit v0.12


From 5512a5f1a932d4d906f3bbecf1932863bcac7b2b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 11:22:57 -0800
Subject: removed useless `(1 && ...)` condition

as reported by @terrelln
---
 lib/lz4hc.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 1880d53..5882109 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -119,8 +119,9 @@ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip)
 
 /** LZ4HC_countBack() :
  * @return : negative value, nb of common bytes before ip/match */
-static int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
-                           const BYTE* const iMin, const BYTE* const mMin)
+LZ4_FORCE_INLINE
+int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
+                    const BYTE* const iMin, const BYTE* const mMin)
 {
     int back=0;
     while ( (ip+back > iMin)
@@ -264,7 +265,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
             matchIndex -= nextOffset;
-            if (1 && (nextOffset==1)) {
+            if (nextOffset==1) {
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ((pattern & 0xFFFF) == (pattern >> 16)) {   /* is it enough ? */
-- 
cgit v0.12


From c49f66f2adf27af499cf7efacbad656bf30d671a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 11:29:28 -0800
Subject: ensure `pattern` is a 1-byte repetition

---
 lib/lz4hc.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5882109..93017f1 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -268,7 +268,8 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
             if (nextOffset==1) {
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
-                    if ((pattern & 0xFFFF) == (pattern >> 16)) {   /* is it enough ? */
+                    if ( ((pattern & 0xFFFF) == (pattern >> 16))
+                      &  ((pattern & 0xFF)   == (pattern >> 24)) ) {
                         repeat = rep_confirmed;
                         srcPatternLength = LZ4HC_countPattern(ip+4, iHighLimit, pattern) + 4;
                     } else {
-- 
cgit v0.12


From 71fd08c17daa3deee0c2c3c64c8f009729055bad Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 11:33:40 -0800
Subject: removed legacy version of LZ4HC_InsertAndFindBestMatch()

---
 lib/lz4hc.c | 54 +-----------------------------------------------------
 1 file changed, 1 insertion(+), 53 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 93017f1..3498391 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -275,7 +275,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                     } else {
                         repeat = rep_not;
                 }   }
-                if ( (repeat == rep_confirmed)   /* proven repeated pattern (1-2-4) */
+                if ( (repeat == rep_confirmed)
                   && (matchIndex >= dictLimit) ) {   /* same segment only */
                     const BYTE* const matchPtr = base + matchIndex;
                     if (LZ4_read32(matchPtr) == pattern) {  /* good candidate */
@@ -296,7 +296,6 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     return longest;
 }
 
-#if 1
 LZ4_FORCE_INLINE
 int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                 const BYTE* const ip, const BYTE* const iLimit,
@@ -307,57 +306,6 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
 }
 
-#else
-
-LZ4_FORCE_INLINE
-int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
-                                const BYTE* const ip, const BYTE* const iLimit,
-                                const BYTE** matchpos,
-                                const int maxNbAttempts)
-{
-    U16* const chainTable = hc4->chainTable;
-    U32* const HashTable = hc4->hashTable;
-    const BYTE* const base = hc4->base;
-    const BYTE* const dictBase = hc4->dictBase;
-    const U32 dictLimit = hc4->dictLimit;
-    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
-    U32 matchIndex;
-    int nbAttempts = maxNbAttempts;
-    size_t ml = 0;
-
-    /* HC4 match finder */
-    LZ4HC_Insert(hc4, ip);
-    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
-
-    while ((matchIndex>=lowLimit) && (nbAttempts)) {
-        nbAttempts--;
-        if (matchIndex >= dictLimit) {
-            const BYTE* const match = base + matchIndex;
-            if ( (*(match+ml) == *(ip+ml))   /* can be longer */
-               && (LZ4_read32(match) == LZ4_read32(ip)) )
-            {
-                size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
-                if (mlt > ml) { ml = mlt; *matchpos = match; }
-            }
-        } else {
-            const BYTE* const match = dictBase + matchIndex;
-            if (LZ4_read32(match) == LZ4_read32(ip)) {
-                size_t mlt;
-                const BYTE* vLimit = ip + (dictLimit - matchIndex);
-                if (vLimit > iLimit) vLimit = iLimit;
-                mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
-                if ((ip+mlt == vLimit) && (vLimit < iLimit))
-                    mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
-                if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
-            }
-        }
-        matchIndex -= DELTANEXTU16(chainTable, matchIndex);
-    }
-
-    return (int)ml;
-}
-#endif
-
 
 
 typedef enum {
-- 
cgit v0.12


From 897f5e9834ed4e8b4e34380b12a6b9e7b911af90 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 17:37:31 -0800
Subject: removed the ip++ at the beginning of block

The first byte used to be skipped
to avoid a infinite self-comparison.
This is no longer necessary, since init() ensures that index starts at 64K.

The first byte is also useless to search when each block is independent,
but it's no longer the case when blocks are linked.

Removing the first-byte-skip saves
about 10 bytes / MB on files compressed with -BD4 (linked blocks 64Kb),
which feels correct as each MB has 16 blocks of 64KB.
---
 lib/lz4hc.c  | 2 --
 lib/lz4opt.h | 1 -
 2 files changed, 3 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 3498391..5e2dd2a 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -429,8 +429,6 @@ static int LZ4HC_compress_hashChain (
     if (limit == limitedDestSize) oend -= LASTLITERALS;                  /* Hack for support limitations LZ4 decompressor */
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
-    ip++;
-
     /* Main Loop */
     while (ip < mflimit) {
         ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index dd22b7a..03ab825 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -127,7 +127,6 @@ static int LZ4HC_compress_optimal (
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
     if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
-    ip++;
 
     /* Main Loop */
     assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
-- 
cgit v0.12


From b07d36245a52048df233a90c382e523d4560fc64 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Nov 2017 17:58:59 -0800
Subject: fixed LZ4HC_reverseCountPattern()

for multi-bytes patterns
(which is not useful for the time being)
---
 lib/lz4hc.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 5e2dd2a..02eafaf 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -173,12 +173,11 @@ static unsigned LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow
         if (LZ4_read32(ip-4) != pattern) break;
         ip -= 4;
     }
-    while (likely(ip>iLow)) {
-        const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
-        if (ip[-1] != *bytePtr) break;
-        ip--; bytePtr--;
-    }
-
+    {   const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianess */
+        while (likely(ip>iLow)) {
+            if (ip[-1] != *bytePtr) break;
+            ip--; bytePtr--;
+    }   }
     return (unsigned)(iStart - ip);
 }
 
-- 
cgit v0.12


From fa03a9d3d983c0bc4a6ea843493d85d434188090 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 8 Nov 2017 08:42:59 -0800
Subject: added code comments

---
 lib/lz4hc.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 02eafaf..b8d8a78 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -223,7 +223,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
                 if (LZ4_read32(matchPtr) == pattern) {
                     int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
     #if 0
-                    /* more generic but unfortunately slower ... */
+                    /* more generic but unfortunately slower on clang */
                     int const back = LZ4HC_countBack(ip, matchPtr, iLowLimit, lowPrefixPtr);
     #else
                     int back = 0;
@@ -297,11 +297,14 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
 LZ4_FORCE_INLINE
 int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
-                                const BYTE* const ip, const BYTE* const iLimit,
-                                const BYTE** matchpos,
-                                const int maxNbAttempts)
+                                 const BYTE* const ip, const BYTE* const iLimit,
+                                 const BYTE** matchpos,
+                                 const int maxNbAttempts)
 {
     const BYTE* uselessPtr = ip;
+    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
+     * but this won't be the case here, as we define iLowLimit==ip,
+     * so LZ4HC_InsertAndGetWiderMatch() will not be allowed to search past ip */
     return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
 }
 
@@ -430,7 +433,7 @@ static int LZ4HC_compress_hashChain (
 
     /* Main Loop */
     while (ip < mflimit) {
-        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts);
         if (ml
Date: Wed, 8 Nov 2017 17:11:51 -0800
Subject: lz4opt: simplified match finder invocation to LZ4HC_FindLongerMatch()

---
 lib/lz4hc.c  |  2 +-
 lib/lz4opt.h | 31 +++++++++++--------------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b8d8a78..60690a0 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -304,7 +304,7 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index tabl
     const BYTE* uselessPtr = ip;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
-     * so LZ4HC_InsertAndGetWiderMatch() will not be allowed to search past ip */
+     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
     return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
 }
 
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 03ab825..6db8586 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -70,32 +70,23 @@ LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen)
 /*-*************************************
 *  Match finder
 ***************************************/
-
-LZ4_FORCE_INLINE
-int LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,   /* Index table will be updated */
-                        const BYTE* const ip, const BYTE* const iHighLimit,
-                        int longest,
-                        const BYTE** matchpos,
-                        const int maxNbAttempts)
-{
-    const BYTE* uselessPtr = ip;
-    return LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, longest, matchpos, &uselessPtr, maxNbAttempts);
-}
-
 typedef struct {
     int off;
     int len;
 } LZ4HC_match_t;
 
 LZ4_FORCE_INLINE
-LZ4HC_match_t LZ4HC_HashChain_GetAllMatches (LZ4HC_CCtx_internal* const ctx,
-                        const BYTE* const ip, const BYTE* const iHighLimit,
-                        size_t best_mlen, int nbSearches)
+LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
+                        const BYTE* ip, const BYTE* const iHighLimit,
+                        int minLen, int nbSearches)
 {
     LZ4HC_match_t match = { 0 , 0 };
     const BYTE* matchPtr = NULL;
-    int matchLength = LZ4HC_FindLongerMatch(ctx, ip, iHighLimit, (int)best_mlen, &matchPtr, nbSearches);
-    if ((size_t)matchLength <= best_mlen) return match;
+    /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
+     * but this won't be the case here, as we define iLowLimit==ip,
+     * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
+    int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, nbSearches);
+    if (matchLength <= minLen) return match;
     match.len = matchLength;
     match.off = (int)(ip-matchPtr);
     return match;
@@ -135,7 +126,7 @@ static int LZ4HC_compress_optimal (
         int best_mlen, best_off;
         int cur, last_match_pos = 0;
 
-        LZ4HC_match_t const firstMatch = LZ4HC_HashChain_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
+        LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches);
         if (firstMatch.len==0) { ip++; continue; }
 
         if ((size_t)firstMatch.len > sufficient_len) {
@@ -199,10 +190,10 @@ static int LZ4HC_compress_optimal (
 
             DEBUGLOG(7, "search at rPos:%u", cur);
             if (fullUpdate)
-                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
+                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches);
             else
                 /* only test matches of minimum length; slightly faster, but misses a few bytes */
-                newMatch = LZ4HC_HashChain_GetAllMatches(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
+                newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches);
             if (!newMatch.len) continue;
 
             if ( ((size_t)newMatch.len > sufficient_len)
-- 
cgit v0.12


From 63f6039fb37bd561c776a1fc4eb04bd9ea46dc83 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 8 Nov 2017 17:47:24 -0800
Subject: added constant TRAILING_LITERALS

which is more explicit than its value `3`.
reported by @terrelln
---
 lib/lz4opt.h | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 6db8586..7f74df9 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -105,7 +105,8 @@ static int LZ4HC_compress_optimal (
     int const fullUpdate
     )
 {
-    LZ4HC_optimal_t opt[LZ4_OPT_NUM + 3];   /* this uses a bit too much stack memory to my taste ... */
+#define TRAILING_LITERALS 3
+    LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS];   /* this uses a bit too much stack memory to my taste ... */
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -165,7 +166,7 @@ static int LZ4HC_compress_optimal (
         }   }
         last_match_pos = firstMatch.len;
         {   int addLit;
-            for (addLit = 1; addLit <= 3; addLit ++) {
+            for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
                 opt[last_match_pos+addLit].mlen = 1; /* literal */
                 opt[last_match_pos+addLit].off = 0;
                 opt[last_match_pos+addLit].litlen = addLit;
@@ -183,7 +184,7 @@ static int LZ4HC_compress_optimal (
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
                     cur, opt[cur].price, opt[cur+1].price, cur+1);
             if (fullUpdate) {
-                if ((opt[cur+1].price <= opt[cur].price) && (opt[cur+4].price < opt[cur].price+3)) continue;
+                if ((opt[cur+1].price <= opt[cur].price) && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/)) continue;
             } else {
                 if (opt[cur+1].price <= opt[cur].price) continue;
             }
@@ -241,7 +242,7 @@ static int LZ4HC_compress_optimal (
                         price = opt[cur].price + LZ4HC_sequencePrice(0, ml);
                     }
 
-                    if (pos > last_match_pos+3 || price <= opt[pos].price) {
+                    if (pos > last_match_pos+TRAILING_LITERALS || price <= opt[pos].price) {
                         DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)",
                                     pos, price, ml);
                         assert(pos < LZ4_OPT_NUM);
@@ -255,7 +256,7 @@ static int LZ4HC_compress_optimal (
             }   }   }
             /* complete following positions with literals */
             {   int addLit;
-                for (addLit = 1; addLit <= 3; addLit ++) {
+                for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) {
                     opt[last_match_pos+addLit].mlen = 1; /* literal */
                     opt[last_match_pos+addLit].off = 0;
                     opt[last_match_pos+addLit].litlen = addLit;
-- 
cgit v0.12


From dc3ed5b6a7d9cc7d251a1942558e37793d5575b8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 8 Nov 2017 17:56:20 -0800
Subject: added code comments

---
 lib/lz4opt.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 7f74df9..9917851 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -184,8 +184,13 @@ static int LZ4HC_compress_optimal (
             DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u",
                     cur, opt[cur].price, opt[cur+1].price, cur+1);
             if (fullUpdate) {
-                if ((opt[cur+1].price <= opt[cur].price) && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/)) continue;
+                /* not useful to search here if next position has same (or lower) cost */
+                if ( (opt[cur+1].price <= opt[cur].price)
+                  /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */
+                  && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) )
+                    continue;
             } else {
+                /* not useful to search here if next position has same (or lower) cost */
                 if (opt[cur+1].price <= opt[cur].price) continue;
             }
 
-- 
cgit v0.12


From b48839b41c10fd4246e7b9849b115785c69fe837 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 13 Nov 2017 15:54:55 -0800
Subject: updated NEWS

---
 NEWS | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index ff20e7e..89398b9 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,12 @@
 v1.8.1
+perf : faster and stronger ultra modes (level 11 and 12)
+perf : slightly faster compression and decompression speed
+perf : fix bad degenerative case, reported by @c-morgenstern
 fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid)
-install: fix : correct man page directory (#387), reported by Stuart Cardall (@itoffshore)
+cli : support for dictionary compression (`-D`), by Felix Handte @felixhandte
 build  : `make` and `make test` compatible with `-jX`, reported by @mwgamera
+build  : can control LZ4LIB_VISIBILITY macro, by @mikir
+install: fix man page directory (#387), reported by Stuart Cardall (@itoffshore)
 
 v1.8.0
 cli : fix : do not modify /dev/null permissions, reported by @Maokaman1
-- 
cgit v0.12


From da8bed4b01fc1221681a8d0bce768444200e91e3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 20 Nov 2017 10:27:05 -0800
Subject: API : changed a few variables' names for clarity

updated relevant doc.
This patch has no impact on ABI/API, nor on binary generation.
---
 doc/lz4_manual.html | 86 +++++++++++++++++++++++++--------------------------
 lib/lz4.h           | 88 ++++++++++++++++++++++++++---------------------------
 2 files changed, 87 insertions(+), 87 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 32d98e2..4c9c7de 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -60,26 +60,26 @@
 
 
Simple Functions

 
-
int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
-
    Compresses 'sourceSize' bytes from buffer 'source'
-    into already allocated 'dest' buffer of size 'maxDestSize'.
-    Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+
int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
+
    Compresses 'srcSize' bytes from buffer 'src'
+    into already allocated 'dst' buffer of size 'dstCapacity'.
+    Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'source' into a more limited 'dest' budget,
+    If the function cannot compress 'src' into a limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dest' content is not valid.
-    This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
-        sourceSize  : Max supported value is LZ4_MAX_INPUT_VALUE
-        maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
-        return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
-              or 0 if compression fails 
+    As a consequence, 'dst' content is not valid.
+    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
+        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
+        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+        return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
+                  or 0 if compression fails 
 


 
-
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-
    compressedSize : is the precise full size of the compressed block.
-    maxDecompressedSize : is the size of destination buffer, which must be already allocated.
-    return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
-             If destination buffer is not large enough, decoding will stop and output an error code (<0).
+
int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
+
    compressedSize : is the exact complete size of the compressed block.
+    dstCapacity : is the size of destination buffer, which must be already allocated.
+    return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
+             If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
              This function is protected against buffer overflow exploits, including malicious data packets.
              It never writes outside output buffer, nor reads outside input buffer.
@@ -97,7 +97,7 @@
               or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
 


 
-
int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+
int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
     The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
     It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
@@ -106,45 +106,45 @@
 


 
 
int LZ4_sizeofState(void);
-int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
    Same compression function, just using an externally allocated memory space to store compression state.
     Use LZ4_sizeofState() to know how much memory must be allocated,
     and allocate it on 8-bytes boundaries (using malloc() typically).
     Then, provide it as 'void* state' to compression function.
 


 
-
int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
-
    Reverse the logic, by compressing as much data as possible from 'source' buffer
-    into already allocated buffer 'dest' of size 'targetDestSize'.
-    This function either compresses the entire 'source' content into 'dest' if it's large enough,
-    or fill 'dest' buffer completely with as much data as possible from 'source'.
-        *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
-                         New value is necessarily <= old value.
-        return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
-              or 0 if compression fails
+
int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
+
    Reverse the logic : compresses as much data as possible from 'src' buffer
+    into already allocated buffer 'dst' of size 'targetDestSize'.
+    This function either compresses the entire 'src' content into 'dst' if it's large enough,
+    or fill 'dst' buffer completely with as much data as possible from 'src'.
+        *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
+                      New value is necessarily <= old value.
+        return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
+                 or 0 if compression fails
 


 
-
int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
-
    originalSize : is the original and therefore uncompressed size
+
int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
+
    originalSize : is the original uncompressed size
     return : the number of bytes read from the source buffer (in other words, the compressed size)
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
-    note : This function fully respect memory boundaries for properly formed compressed data.
+             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+    note : This function respects memory boundaries for *properly formed* compressed data.
            It is a bit faster than LZ4_decompress_safe().
            However, it does not provide any protection against intentionally modified data stream (malicious input).
            Use this function in trusted environment only (data to decode comes from a trusted source).
 


 
-
int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
-
    This function decompress a compressed block of size 'compressedSize' at position 'source'
-    into destination buffer 'dest' of size 'maxDecompressedSize'.
-    The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
-    reducing decompression time.
-    return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
-       Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+
int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
+
    This function decompress a compressed block of size 'srcSize' at position 'src'
+    into destination buffer 'dst' of size 'dstCapacity'.
+    The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
+    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+   @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
+       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
              Always control how many bytes were decoded.
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
 


 
 
Streaming Compression Functions

@@ -210,8 +210,8 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 


 
-
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
 
  These decoding functions allow decompression of consecutive blocks in "streaming" mode.
   A block is an unsplittable entity, it must be presented entirely to a decompression function.
   Decompression functions only accept one block at a time.
@@ -231,8 +231,8 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
   and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 


 
-
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
-int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
 
  These decoding functions work the same as
   a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
   They are stand-alone, and don't need an LZ4_streamDecode_t structure.
diff --git a/lib/lz4.h b/lib/lz4.h
index 509d942..3cf6c8d 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -124,30 +124,30 @@ LZ4LIB_API const char* LZ4_versionString (void);   /**< library version string;
 *  Simple Functions
 **************************************/
 /*! LZ4_compress_default() :
-    Compresses 'sourceSize' bytes from buffer 'source'
-    into already allocated 'dest' buffer of size 'maxDestSize'.
-    Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+    Compresses 'srcSize' bytes from buffer 'src'
+    into already allocated 'dst' buffer of size 'dstCapacity'.
+    Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
     It also runs faster, so it's a recommended setting.
-    If the function cannot compress 'source' into a more limited 'dest' budget,
+    If the function cannot compress 'src' into a limited 'dst' budget,
     compression stops *immediately*, and the function result is zero.
-    As a consequence, 'dest' content is not valid.
-    This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
-        sourceSize  : Max supported value is LZ4_MAX_INPUT_VALUE
-        maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
-        return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
-              or 0 if compression fails */
-LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+    As a consequence, 'dst' content is not valid.
+    This function never writes outside 'dst' buffer, nor read outside 'source' buffer.
+        srcSize : supported max value is LZ4_MAX_INPUT_VALUE
+        dstCapacity : full or partial size of buffer 'dst' (which must be already allocated)
+        return  : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
+                  or 0 if compression fails */
+LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
 
 /*! LZ4_decompress_safe() :
-    compressedSize : is the precise full size of the compressed block.
-    maxDecompressedSize : is the size of destination buffer, which must be already allocated.
-    return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
-             If destination buffer is not large enough, decoding will stop and output an error code (<0).
+    compressedSize : is the exact complete size of the compressed block.
+    dstCapacity : is the size of destination buffer, which must be already allocated.
+    return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
+             If destination buffer is not large enough, decoding will stop and output an error code (negative value).
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
              This function is protected against buffer overflow exploits, including malicious data packets.
              It never writes outside output buffer, nor reads outside input buffer.
 */
-LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
 
 
 /*-************************************
@@ -176,7 +176,7 @@ LZ4_compress_fast() :
     An acceleration value of "1" is the same as regular LZ4_compress_default()
     Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
 */
-LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
 /*!
@@ -187,49 +187,49 @@ LZ4_compress_fast_extState() :
     Then, provide it as 'void* state' to compression function.
 */
 LZ4LIB_API int LZ4_sizeofState(void);
-LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
 
 /*!
 LZ4_compress_destSize() :
-    Reverse the logic, by compressing as much data as possible from 'source' buffer
-    into already allocated buffer 'dest' of size 'targetDestSize'.
-    This function either compresses the entire 'source' content into 'dest' if it's large enough,
-    or fill 'dest' buffer completely with as much data as possible from 'source'.
-        *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
-                         New value is necessarily <= old value.
-        return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
-              or 0 if compression fails
+    Reverse the logic : compresses as much data as possible from 'src' buffer
+    into already allocated buffer 'dst' of size 'targetDestSize'.
+    This function either compresses the entire 'src' content into 'dst' if it's large enough,
+    or fill 'dst' buffer completely with as much data as possible from 'src'.
+        *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
+                      New value is necessarily <= old value.
+        return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
+                 or 0 if compression fails
 */
-LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
+LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
 
 
 /*!
-LZ4_decompress_fast() :
-    originalSize : is the original and therefore uncompressed size
+LZ4_decompress_fast() : (unsafe!!)
+    originalSize : is the original uncompressed size
     return : the number of bytes read from the source buffer (in other words, the compressed size)
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
-    note : This function fully respect memory boundaries for properly formed compressed data.
+             Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes.
+    note : This function respects memory boundaries for *properly formed* compressed data.
            It is a bit faster than LZ4_decompress_safe().
            However, it does not provide any protection against intentionally modified data stream (malicious input).
            Use this function in trusted environment only (data to decode comes from a trusted source).
 */
-LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
+LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
 
 /*!
 LZ4_decompress_safe_partial() :
-    This function decompress a compressed block of size 'compressedSize' at position 'source'
-    into destination buffer 'dest' of size 'maxDecompressedSize'.
-    The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
-    reducing decompression time.
-    return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
-       Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+    This function decompress a compressed block of size 'srcSize' at position 'src'
+    into destination buffer 'dst' of size 'dstCapacity'.
+    The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that.
+    However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity).
+   @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity)
+       Note : this number can be < 'targetOutputSize' should the compressed block contain less data.
              Always control how many bytes were decoded.
              If the source stream is detected malformed, the function will stop decoding and return a negative result.
-             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets.
 */
-LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
+LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
 
 
 /*-*********************************************
@@ -321,8 +321,8 @@ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const
  *  Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
  *  and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block.
 */
-LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
+LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
 
 
 /*! LZ4_decompress_*_usingDict() :
@@ -330,8 +330,8 @@ LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecod
  *  a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
  *  They are stand-alone, and don't need an LZ4_streamDecode_t structure.
  */
-LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
-LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
+LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
 
 
 /*^**********************************************
-- 
cgit v0.12


From 6c94c94d4673bd2c0bc3ff80ca17cff8c19a465a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 24 Nov 2017 17:18:46 -0800
Subject: minor updates to examples

see https://github.com/lz4/lz4/commit/810e2ca27b3561e0f6bfa7a88e0fde6faf807064#commitcomment-25810887
---
 examples/HCStreaming_ringBuffer.c      |   6 +-
 examples/blockStreaming_doubleBuffer.c |   2 +-
 examples/blockStreaming_lineByLine.c   |   4 +-
 examples/blockStreaming_ringBuffer.c   |   2 +-
 examples/dictionaryRandomAccess.c      |   2 +-
 examples/frameCompress.c               | 105 +++++++++++++++++----------------
 examples/printVersion.c                |   2 +-
 programs/platform.h                    |  10 ++--
 8 files changed, 67 insertions(+), 66 deletions(-)

diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c
index d49b267..a878577 100644
--- a/examples/HCStreaming_ringBuffer.c
+++ b/examples/HCStreaming_ringBuffer.c
@@ -1,12 +1,12 @@
 // LZ4 HC streaming API example : ring buffer
-// Based on previous work from Takayuki Matsuoka
+// Based on a previous example by Takayuki Matsuoka
 
 
 /**************************************
  * Compiler Options
  **************************************/
-#ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS   /* for MSVC */
+#if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
+#  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
 
diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c
index d02f258..acb3455 100644
--- a/examples/blockStreaming_doubleBuffer.c
+++ b/examples/blockStreaming_doubleBuffer.c
@@ -2,7 +2,7 @@
 // Copyright : Takayuki Matsuoka
 
 
-#ifdef _MSC_VER    /* Visual Studio */
+#if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
 #  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c
index f449aa3..677c426 100644
--- a/examples/blockStreaming_lineByLine.c
+++ b/examples/blockStreaming_lineByLine.c
@@ -1,8 +1,8 @@
 // LZ4 streaming API example : line-by-line logfile compression
-// Copyright : Takayuki Matsuoka
+// by Takayuki Matsuoka
 
 
-#ifdef _MSC_VER    /* Visual Studio */
+#if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
 #  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c
index dec78af..0b6a3ce 100644
--- a/examples/blockStreaming_ringBuffer.c
+++ b/examples/blockStreaming_ringBuffer.c
@@ -5,7 +5,7 @@
 /**************************************
  * Compiler Options
  **************************************/
-#ifdef _MSC_VER    /* Visual Studio */
+#if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
 #  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c
index 6acf99b..291fd08 100644
--- a/examples/dictionaryRandomAccess.c
+++ b/examples/dictionaryRandomAccess.c
@@ -1,6 +1,6 @@
 // LZ4 API example : Dictionary Random Access
 
-#ifdef _MSC_VER    /* Visual Studio */
+#if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
 #  define _CRT_SECURE_NO_WARNINGS
 #  define snprintf sprintf_s
 #endif
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 80db90e..d66a8dc 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -21,17 +21,15 @@ static const LZ4F_preferences_t lz4_preferences = {
 };
 
 static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
-    LZ4F_errorCode_t r;
+    size_t r=1;  /* function result; 1 == error, default (early exit) */
     LZ4F_compressionContext_t ctx;
     char *src, *buf = NULL;
-    size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
+    size_t size, count_in = 0, count_out, offset = 0, frame_size;
 
-    r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
-    if (LZ4F_isError(r)) {
+    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
         printf("Failed to create context: error %zu\n", r);
         return 1;
     }
-    r = 1;  /* function result; 1 == error, by default (early exit) */
 
     src = malloc(BUF_SIZE);
     if (!src) {
@@ -40,41 +38,45 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_o
     }
 
     frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
-    size =  frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
+    size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
     buf = malloc(size);
     if (!buf) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
-    if (LZ4F_isError(n)) {
-        printf("Failed to start compression: error %zu\n", n);
-        goto cleanup;
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+        if (LZ4F_isError(headerSize)) {
+            printf("Failed to start compression: error %zu\n", headerSize);
+            goto cleanup;
+        }
+        offset = count_out = headerSize;
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
     }
 
-    printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
 
     for (;;) {
-        k = fread(src, 1, BUF_SIZE, in);
-        if (k == 0)
+        size_t const readSize = fread(src, 1, BUF_SIZE, in);
+        if (readSize == 0)
             break;
-        count_in += k;
+        count_in += readSize;
 
-        n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
-        if (LZ4F_isError(n)) {
-            printf("Compression failed: error %zu\n", n);
-            goto cleanup;
+        {   size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
+            if (LZ4F_isError(compressedSize)) {
+                printf("Compression failed: error %zu\n", compressedSize);
+                goto cleanup;
+            }
+            offset += compressedSize;
+            count_out += compressedSize;
         }
 
-        offset += n;
-        count_out += n;
         if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
+            size_t writtenSize;
             printf("Writing %zu bytes\n", offset);
 
-            k = fwrite(buf, 1, offset, out);
-            if (k < offset) {
-                if (ferror(out))
+            writtenSize = fwrite(buf, 1, offset, out);
+            if (writtenSize < offset) {
+                if (ferror(out))  /* note : ferror() must follow fwrite */
                     printf("Write failed\n");
                 else
                     printf("Short write\n");
@@ -85,31 +87,31 @@ static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_o
         }
     }
 
-    n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
-    if (LZ4F_isError(n)) {
-        printf("Failed to end compression: error %zu\n", n);
-        goto cleanup;
+    {   size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
+        if (LZ4F_isError(compressedSize)) {
+            printf("Failed to end compression: error %zu\n", compressedSize);
+            goto cleanup;
+        }
+        offset += compressedSize;
+        count_out += compressedSize;
     }
 
-    offset += n;
-    count_out += n;
     printf("Writing %zu bytes\n", offset);
-
-    k = fwrite(buf, 1, offset, out);
-    if (k < offset) {
-        if (ferror(out))
-            printf("Write failed\n");
-        else
-            printf("Short write\n");
-        goto cleanup;
-    }
+    {   size_t const writtenSize = fwrite(buf, 1, offset, out);
+        if (writtenSize < offset) {
+            if (ferror(out))
+                printf("Write failed\n");
+            else
+                printf("Short write\n");
+            goto cleanup;
+    }   }
 
     *size_in = count_in;
     *size_out = count_out;
-    r = 0;
+    r = 0;   /* success */
+
  cleanup:
-    if (ctx)
-        LZ4F_freeCompressionContext(ctx);
+    LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
     free(buf);
     return r;
@@ -128,28 +130,27 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     }
 }
 
-static size_t decompress_file(FILE *in, FILE *out) {
+static size_t decompress_file(FILE* in, FILE* out) {
     void* const src = malloc(BUF_SIZE);
     void* dst = NULL;
     size_t dstCapacity = 0;
-    LZ4F_dctx *dctx = NULL;
-    size_t ret;
+    LZ4F_dctx* dctx = NULL;
+    size_t ret = 1;
 
     /* Initialization */
     if (!src) { perror("decompress_file(src)"); goto cleanup; }
-    ret = LZ4F_createDecompressionContext(&dctx, 100);
-    if (LZ4F_isError(ret)) {
-        printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret));
-        goto cleanup;
-    }
+    {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
+        if (LZ4F_isError(dctxStatus)) {
+            printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
+            goto cleanup;
+    }   }
 
     /* Decompression */
-    ret = 1;
     while (ret != 0) {
         /* Load more input */
         size_t srcSize = fread(src, 1, BUF_SIZE, in);
-        void* srcPtr = src;
-        void* srcEnd = srcPtr + srcSize;
+        const void* srcPtr = src;
+        const void* const srcEnd = srcPtr + srcSize;
         if (srcSize == 0 || ferror(in)) {
             printf("Decompress: not enough input or error reading file\n");
             goto cleanup;
diff --git a/examples/printVersion.c b/examples/printVersion.c
index 8607139..7af318a 100644
--- a/examples/printVersion.c
+++ b/examples/printVersion.c
@@ -1,5 +1,5 @@
 // LZ4 trivial example : print Library version number
-// Copyright : Takayuki Matsuoka & Yann Collet
+// by Takayuki Matsuoka
 
 
 #include 
diff --git a/programs/platform.h b/programs/platform.h
index 66491b6..db2efac 100644
--- a/programs/platform.h
+++ b/programs/platform.h
@@ -30,10 +30,10 @@ extern "C" {
 *  Compiler Options
 ****************************************/
 #if defined(_MSC_VER)
-#  define _CRT_SECURE_NO_WARNINGS   /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
-#  define _CRT_SECURE_NO_DEPRECATE  /* VS2005 - must be declared before  and  */ 
-#  if (_MSC_VER <= 1800)            /* (1800 = Visual Studio 2013) */
-#    define snprintf sprintf_s      /* snprintf unsupported by Visual <= 2013 */
+#  define _CRT_SECURE_NO_WARNINGS    /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
+#  if (_MSC_VER <= 1800)             /* (1800 = Visual Studio 2013) */
+#    define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before  and  */
+#    define snprintf sprintf_s       /* snprintf unsupported by Visual <= 2013 */
 #  endif
 #endif
 
@@ -60,7 +60,7 @@ extern "C" {
 *  Turn on Large Files support (>4GB) for 32-bit Linux/Unix
 ***********************************************************/
 #if !defined(__64BIT__) || defined(__MINGW32__)       /* No point defining Large file for 64 bit but MinGW-w64 requires it */
-#  if !defined(_FILE_OFFSET_BITS)   
+#  if !defined(_FILE_OFFSET_BITS)
 #    define _FILE_OFFSET_BITS 64                      /* turn off_t into a 64-bit type for ftello, fseeko */
 #  endif
 #  if !defined(_LARGEFILE_SOURCE)                     /* obsolete macro, replaced with _FILE_OFFSET_BITS */
-- 
cgit v0.12


From 42a31aee50ec325fbba93ce405ab83c25fc710e1 Mon Sep 17 00:00:00 2001
From: Alice Atlas 
Date: Mon, 4 Dec 2017 16:07:31 -0500
Subject: Fix bug which could sometimes result in the lz4 cli chmodding
 /dev/null if running as root

---
 programs/lz4io.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 57434f7..9bf4e93 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -616,7 +616,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
 
     /* Copy owner, file permissions and modification time */
     {   stat_t statbuf;
-        if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf))
+        if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && strcmp (dstFileName, nulmark) && UTIL_getFileStat(srcFileName, &statbuf))
             UTIL_setFileStat(dstFileName, &statbuf);
     }
 
-- 
cgit v0.12


From 6bbe45e1b848a08fe0e74bbe12d4863a6f6fdd91 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 4 Dec 2017 17:10:23 -0800
Subject: remove `register` keyword

deprecated in newer C++ versions,
and dubious utility
---
 lib/lz4.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/lz4.c b/lib/lz4.c
index cc76eba..213b085 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -85,16 +85,6 @@
 #endif
 
 
-/*
- * register is ignored when the code built with a C++-17 compiler
- * Remove the keyword when built with C++-17 to silent the warning
- */
-#if defined(__cplusplus) &&  __cplusplus > 201402L
-#  define REGISTER
-#else
-#  define REGISTER register
-#endif
-
 
 /*-************************************
 *  Dependency
@@ -350,7 +340,7 @@ static int g_debuglog_enable = 1;
 /*-************************************
 *  Common functions
 **************************************/
-static unsigned LZ4_NbCommonBytes (REGISTER reg_t val)
+static unsigned LZ4_NbCommonBytes (reg_t val)
 {
     if (LZ4_isLittleEndian()) {
         if (sizeof(val)==8) {
@@ -361,7 +351,14 @@ static unsigned LZ4_NbCommonBytes (REGISTER reg_t val)
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
             return (__builtin_ctzll((U64)val) >> 3);
 #       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                                                     0, 3, 1, 3, 1, 4, 2, 7,
+                                                     0, 2, 3, 6, 1, 5, 3, 5,
+                                                     1, 3, 4, 4, 2, 5, 6, 7,
+                                                     7, 0, 1, 2, 3, 3, 4, 6,
+                                                     2, 6, 5, 5, 3, 4, 5, 6,
+                                                     7, 1, 2, 4, 6, 4, 4, 5,
+                                                     7, 2, 6, 5, 7, 6, 7, 7 };
             return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
 #       endif
         } else /* 32 bits */ {
@@ -372,7 +369,10 @@ static unsigned LZ4_NbCommonBytes (REGISTER reg_t val)
 #       elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT)
             return (__builtin_ctz((U32)val) >> 3);
 #       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                                                     3, 2, 2, 1, 3, 2, 0, 1,
+                                                     3, 3, 1, 2, 2, 2, 2, 0,
+                                                     3, 1, 2, 0, 1, 0, 1, 1 };
             return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
 #       endif
         }
-- 
cgit v0.12


From 9be957353393b1716c0597b402f4859c53ea6269 Mon Sep 17 00:00:00 2001
From: Max Risuhin 
Date: Mon, 11 Dec 2017 14:57:19 +0200
Subject: EnableWholeProgramOptimization and UseStaticCRT msbuild custom
 properties

---
 appveyor.yml                                      | 2 +-
 visual/VS2010/datagen/datagen.vcxproj             | 8 ++++++--
 visual/VS2010/frametest/frametest.vcxproj         | 8 ++++++--
 visual/VS2010/fullbench-dll/fullbench-dll.vcxproj | 8 ++++++--
 visual/VS2010/fullbench/fullbench.vcxproj         | 8 ++++++--
 visual/VS2010/fuzzer/fuzzer.vcxproj               | 8 ++++++--
 visual/VS2010/liblz4-dll/liblz4-dll.vcxproj       | 8 ++++++--
 visual/VS2010/liblz4/liblz4.vcxproj               | 8 ++++++--
 visual/VS2010/lz4/lz4.vcxproj                     | 8 ++++++--
 9 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 93c1101..056719a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -82,7 +82,7 @@ build_script:
       ECHO *** &&
       ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj
index aaf81ad..e24f961 100644
--- a/visual/VS2010/datagen/datagen.vcxproj
+++ b/visual/VS2010/datagen/datagen.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -107,6 +108,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -124,6 +126,7 @@
       WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -144,6 +147,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj
index 76d12c9..3196768 100644
--- a/visual/VS2010/frametest/frametest.vcxproj
+++ b/visual/VS2010/frametest/frametest.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -107,6 +108,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -124,6 +126,7 @@
       WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -144,6 +147,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
index c10552a..8f503f5 100644
--- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
+++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -109,6 +110,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -128,6 +130,7 @@
       WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -150,6 +153,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj
index e2d95c9..aa67431 100644
--- a/visual/VS2010/fullbench/fullbench.vcxproj
+++ b/visual/VS2010/fullbench/fullbench.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -107,6 +108,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -124,6 +126,7 @@
       WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -144,6 +147,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj
index 85d6c9b..21cbf56 100644
--- a/visual/VS2010/fuzzer/fuzzer.vcxproj
+++ b/visual/VS2010/fuzzer/fuzzer.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -107,6 +108,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -124,6 +126,7 @@
       WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -144,6 +147,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
diff --git a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
index 389f13c..56ec3b9 100644
--- a/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
+++ b/visual/VS2010/liblz4-dll/liblz4-dll.vcxproj
@@ -40,14 +40,14 @@
   
     DynamicLibrary
     false
-    true
     Unicode
+    true
   
   
     DynamicLibrary
     false
-    true
     Unicode
+    true
   
   
   
@@ -96,6 +96,7 @@
       WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       true
@@ -111,6 +112,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       true
@@ -127,6 +129,7 @@
       WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       true
@@ -146,6 +149,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       true
diff --git a/visual/VS2010/liblz4/liblz4.vcxproj b/visual/VS2010/liblz4/liblz4.vcxproj
index a0b8000..61ea159 100644
--- a/visual/VS2010/liblz4/liblz4.vcxproj
+++ b/visual/VS2010/liblz4/liblz4.vcxproj
@@ -39,14 +39,14 @@
   
     StaticLibrary
     false
-    true
     Unicode
+    true
   
   
     StaticLibrary
     false
-    true
     Unicode
+    true
   
   
   
@@ -95,6 +95,7 @@
       WIN32;_DEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       true
@@ -110,6 +111,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       true
@@ -126,6 +128,7 @@
       WIN32;NDEBUG;LZ4_DLL_EXPORT=1;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       true
@@ -145,6 +148,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       true
diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj
index 693e121..de7a714 100644
--- a/visual/VS2010/lz4/lz4.vcxproj
+++ b/visual/VS2010/lz4/lz4.vcxproj
@@ -39,14 +39,14 @@
   
     Application
     false
-    true
     Unicode
+    true
   
   
     Application
     false
-    true
     Unicode
+    true
   
   
   
@@ -91,6 +91,7 @@
       WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
+      MultiThreadedDebug
     
     
       Console
@@ -108,6 +109,7 @@
       true
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreadedDebug
     
     
       Console
@@ -126,6 +128,7 @@
       WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
+      MultiThreaded
     
     
       Console
@@ -147,6 +150,7 @@
       false
       true
       /analyze:stacksize295252 %(AdditionalOptions)
+      MultiThreaded
     
     
       Console
-- 
cgit v0.12


From 55da545e7ac7b1335beac65aba2a1a30cef118ba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 20 Dec 2017 14:14:01 +0100
Subject: new level 10

lz4opt is only competitive vs lz4hc level 10.
Below that level, it doesn't match the speed / compression effectiveness of regular hc parser.

This patch propose to extend lz4opt to levels 10-12.
The new level 10 tend to compress a bit better and a bit faster than previous one (mileage vary depending on file)

The only downside is that `limitedDestSize` mode is now limited to max level 9 (vs 10),
since it's only compatible with regular HC parser.
(Note : I suspect it's possible to convert lz4opt to support it too, but haven't spent time into it).
---
 lib/lz4hc.c | 52 ++++++++++++++++++++++++++++++++++++++--------------
 lib/lz4hc.h |  2 +-
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 60690a0..388eb40 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -619,22 +619,46 @@ static int LZ4HC_compress_generic (
     limitedOutput_directive limit
     )
 {
+    typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
+    typedef struct {
+        lz4hc_strat_e strat;
+        U32 nbSearches;
+        U32 targetLength;
+    } cParams_t;
+    static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {
+        { lz4hc,    2, 16 },  /* 0, unused */
+        { lz4hc,    2, 16 },  /* 1, unused */
+        { lz4hc,    2, 16 },  /* 2, unused */
+        { lz4hc,    4, 16 },  /* 3 */
+        { lz4hc,    8, 16 },  /* 4 */
+        { lz4hc,   16, 16 },  /* 5 */
+        { lz4hc,   32, 16 },  /* 6 */
+        { lz4hc,   64, 16 },  /* 7 */
+        { lz4hc,  128, 16 },  /* 8 */
+        { lz4hc,  256, 16 },  /* 9 */
+        { lz4opt,  96, 64 },  /*10==LZ4HC_CLEVEL_OPT_MIN*/
+        { lz4opt, 512,128 },  /*11 */
+        { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
+    };
+
     ctx->end += *srcSizePtr;
     if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe something to review */
-    if (cLevel > 9) {
-        if (limit == limitedDestSize) cLevel = 10;
-        switch (cLevel) {
-            case 10:
-                return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1<<12, limit);
-            case 11:
-                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 512, 128, 0);
-            default:
-                /* fall-through */
-            case 12:
-                return LZ4HC_compress_optimal(ctx, src, dst, *srcSizePtr, dstCapacity, limit, 1<<13, LZ4_OPT_NUM, 1);
-        }
+    cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
+    if (limit == limitedDestSize)
+        cLevel = MIN(LZ4HC_CLEVEL_OPT_MIN-1, cLevel);  /* no limitedDestSize variant for lz4opt */
+    assert(cLevel >= 0);
+    assert(cLevel <= LZ4HC_CLEVEL_MAX);
+    {   cParams_t const cParam = clTable[cLevel];
+        if (cParam.strat == lz4hc)
+            return LZ4HC_compress_hashChain(ctx,
+                                src, dst, srcSizePtr, dstCapacity,
+                                cParam.nbSearches, limit);
+        assert(cParam.strat == lz4opt);
+        return LZ4HC_compress_optimal(ctx,
+                            src, dst, *srcSizePtr, dstCapacity, limit,
+                            cParam.nbSearches, cParam.targetLength,
+                            cLevel == LZ4HC_CLEVEL_MAX);  /* ultra mode */
     }
-    return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (cLevel-1), limit);  /* levels 1-9 */
 }
 
 
@@ -667,7 +691,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in
 }
 
 /* LZ4_compress_HC_destSize() :
- * only compatible with Hash Chain match finder */
+ * only compatible with regular HC parser */
 int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel)
 {
     LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse;
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index d791062..d41bf42 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -46,7 +46,7 @@ extern "C" {
 /* --- Useful constants --- */
 #define LZ4HC_CLEVEL_MIN         3
 #define LZ4HC_CLEVEL_DEFAULT     9
-#define LZ4HC_CLEVEL_OPT_MIN    11
+#define LZ4HC_CLEVEL_OPT_MIN    10
 #define LZ4HC_CLEVEL_MAX        12
 
 
-- 
cgit v0.12


From 9753ac4c91b926114ae636d1d7103dd24e5c0f57 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 22 Dec 2017 08:07:25 +0100
Subject: conditional pattern analysis

Pattern analysis (currently limited to long ranges of identical bytes)
is actually detrimental to performance
when `nbSearches` is low.

Reason is : `nbSearches` provides a built-in protection for these cases.
The problem with patterns is that they dramatically increase the number of candidates to visit.
But with a low nbSearches, the match finder just aborts early.

In such cases, pattern analysis adds some complexity without reducing total nb of candidates.
It actually increases compression ratio a little bit, by filtering only "good" candidates,
but at a measurable speed cost, so it's not a good trade-off.

This patch makes pattern analysis optional.
It's enabled for levels 8+ only.
---
 lib/lz4hc.c  | 21 ++++++++++++++-------
 lib/lz4opt.h |  4 +++-
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 388eb40..2fff29b 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -191,7 +191,8 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
     int longest,
     const BYTE** matchpos,
     const BYTE** startpos,
-    const int maxNbAttempts)
+    const int maxNbAttempts,
+    const int patternAnalysis)
 {
     U16* const chainTable = hc4->chainTable;
     U32* const HashTable = hc4->hashTable;
@@ -264,7 +265,7 @@ LZ4_FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
 
         {   U32 const nextOffset = DELTANEXTU16(chainTable, matchIndex);
             matchIndex -= nextOffset;
-            if (nextOffset==1) {
+            if (patternAnalysis && nextOffset==1) {
                 /* may be a repeated pattern */
                 if (repeat == rep_untested) {
                     if ( ((pattern & 0xFFFF) == (pattern >> 16))
@@ -299,13 +300,14 @@ LZ4_FORCE_INLINE
 int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4,   /* Index table will be updated */
                                  const BYTE* const ip, const BYTE* const iLimit,
                                  const BYTE** matchpos,
-                                 const int maxNbAttempts)
+                                 const int maxNbAttempts,
+                                 const int patternAnalysis)
 {
     const BYTE* uselessPtr = ip;
     /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos),
      * but this won't be the case here, as we define iLowLimit==ip,
      * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */
-    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts);
+    return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis);
 }
 
 
@@ -403,6 +405,7 @@ static int LZ4HC_compress_hashChain (
     )
 {
     const int inputSize = *srcSizePtr;
+    const int patternAnalysis = (maxNbAttempts > 64);   /* levels 8+ */
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
@@ -433,7 +436,7 @@ static int LZ4HC_compress_hashChain (
 
     /* Main Loop */
     while (ip < mflimit) {
-        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts);
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis);
         if (ml
Date: Fri, 22 Dec 2017 11:54:43 +0100
Subject: /tests programs compiled with LZ4_DEBUG=1

to enable assert() within /lib
---
 tests/Makefile |  5 +++--
 tests/fuzzer.c | 44 +++++++++++++++++++++-----------------------
 2 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 1a907b7..ce8c3fb 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,11 +35,12 @@ PRGDIR  := ../programs
 TESTDIR := versionsTest
 PYTHON  ?= python3
 
+DEBUGFLAGS = -g -DLZ4_DEBUG=1
 CFLAGS  ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
-CFLAGS  += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
+CFLAGS  += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
            -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
            -Wpointer-arith -Wstrict-aliasing=1
-CFLAGS  += $(MOREFLAGS)
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
 FLAGS    = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
 
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ddd293c..365ff79 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -898,7 +898,6 @@ static void FUZ_unitTests(int compressionLevel)
                 }
 
                 dict = dst;
-                //dict = testInput + segStart;
                 dictSize = segSize;
 
                 dst += segSize + 1;
@@ -972,28 +971,27 @@ static void FUZ_unitTests(int compressionLevel)
 #define BSIZE2 16435
 
             /* first block */
-
-                messageSize = BSIZE1;
-                XXH64_update(&xxhOrig, testInput + iNext, messageSize);
-                crcOrig = XXH64_digest(&xxhOrig);
-
-                result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
-
-                result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
-                FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
-
-                XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                { U64 const crcNew = XXH64_digest(&xxhNew);
-                  FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
-
-                /* prepare next message */
-                dNext += messageSize;
-                totalMessageSize += messageSize;
-                messageSize = BSIZE2;
-                iNext = 132000;
-                memcpy(testInput + iNext, testInput + 8, messageSize);
-                if (dNext > dBufferSize) dNext = 0;
+            messageSize = BSIZE1;
+            XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+            crcOrig = XXH64_digest(&xxhOrig);
+
+            result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+            FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+
+            result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
+            FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+
+            XXH64_update(&xxhNew, testVerify + dNext, messageSize);
+            { U64 const crcNew = XXH64_digest(&xxhNew);
+              FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
+
+            /* prepare next message */
+            dNext += messageSize;
+            totalMessageSize += messageSize;
+            messageSize = BSIZE2;
+            iNext = 132000;
+            memcpy(testInput + iNext, testInput + 8, messageSize);
+            if (dNext > dBufferSize) dNext = 0;
 
             while (totalMessageSize < 9 MB) {
                 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
-- 
cgit v0.12


From 7d2f30c7d1519a3a4a199e398e13d9baa21bf84e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 22 Dec 2017 12:47:59 +0100
Subject: lz4opt supports _destSize

no longer limited to level 9
---
 lib/lz4hc.c    | 14 ++++++--------
 lib/lz4opt.h   | 61 +++++++++++++++++++++++++++++++++++++++++-----------------
 tests/fuzzer.c | 15 ++++++++-------
 3 files changed, 57 insertions(+), 33 deletions(-)

diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 388eb40..e6d496c 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -425,10 +425,7 @@ static int LZ4HC_compress_hashChain (
 
     /* init */
     *srcSizePtr = 0;
-    if (limit == limitedDestSize && maxOutputSize < 1) return 0;         /* Impossible to store anything */
-    if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0;              /* Unsupported input size, too large (or negative) */
-
-    if (limit == limitedDestSize) oend -= LASTLITERALS;                  /* Hack for support limitations LZ4 decompressor */
+    if (limit == limitedDestSize) oend -= LASTLITERALS;                  /* Hack for support LZ4 format restriction */
     if (inputSize < LZ4_minLength) goto _last_literals;                  /* Input too small, no compression (all literals) */
 
     /* Main Loop */
@@ -641,11 +638,12 @@ static int LZ4HC_compress_generic (
         { lz4opt,8192, LZ4_OPT_NUM },  /* 12==LZ4HC_CLEVEL_MAX */
     };
 
+    if (limit == limitedDestSize && dstCapacity < 1) return 0;         /* Impossible to store anything */
+    if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;          /* Unsupported input size (too large or negative) */
+
     ctx->end += *srcSizePtr;
     if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT;   /* note : convention is different from lz4frame, maybe something to review */
     cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
-    if (limit == limitedDestSize)
-        cLevel = MIN(LZ4HC_CLEVEL_OPT_MIN-1, cLevel);  /* no limitedDestSize variant for lz4opt */
     assert(cLevel >= 0);
     assert(cLevel <= LZ4HC_CLEVEL_MAX);
     {   cParams_t const cParam = clTable[cLevel];
@@ -655,8 +653,8 @@ static int LZ4HC_compress_generic (
                                 cParam.nbSearches, limit);
         assert(cParam.strat == lz4opt);
         return LZ4HC_compress_optimal(ctx,
-                            src, dst, *srcSizePtr, dstCapacity, limit,
-                            cParam.nbSearches, cParam.targetLength,
+                            src, dst, srcSizePtr, dstCapacity,
+                            cParam.nbSearches, cParam.targetLength, limit,
                             cLevel == LZ4HC_CLEVEL_MAX);  /* ultra mode */
     }
 }
diff --git a/lib/lz4opt.h b/lib/lz4opt.h
index 9917851..58068bf 100644
--- a/lib/lz4opt.h
+++ b/lib/lz4opt.h
@@ -97,11 +97,11 @@ static int LZ4HC_compress_optimal (
     LZ4HC_CCtx_internal* ctx,
     const char* const source,
     char* dst,
-    int inputSize,
+    int* srcSizePtr,
     int dstCapacity,
-    limitedOutput_directive limit,
     int const nbSearches,
     size_t sufficient_len,
+    limitedOutput_directive limit,
     int const fullUpdate
     )
 {
@@ -110,14 +110,17 @@ static int LZ4HC_compress_optimal (
 
     const BYTE* ip = (const BYTE*) source;
     const BYTE* anchor = ip;
-    const BYTE* const iend = ip + inputSize;
+    const BYTE* const iend = ip + *srcSizePtr;
     const BYTE* const mflimit = iend - MFLIMIT;
     const BYTE* const matchlimit = iend - LASTLITERALS;
     BYTE* op = (BYTE*) dst;
-    BYTE* const oend = op + dstCapacity;
+    BYTE* opSaved = (BYTE*) dst;
+    BYTE* oend = op + dstCapacity;
 
     /* init */
     DEBUGLOG(5, "LZ4HC_compress_optimal");
+    *srcSizePtr = 0;
+    if (limit == limitedDestSize) oend -= LASTLITERALS;   /* Hack for support LZ4 format restriction */
     if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
 
     /* Main Loop */
@@ -134,8 +137,9 @@ static int LZ4HC_compress_optimal (
             /* good enough solution : immediate encoding */
             int const firstML = firstMatch.len;
             const BYTE* const matchPos = ip - firstMatch.off;
+            opSaved = op;
             if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) )   /* updates ip, op and anchor */
-                return 0;  /* error */
+                goto _dest_overflow;
             continue;
         }
 
@@ -304,26 +308,47 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
                 rPos += ml;
                 assert(ml >= MINMATCH);
                 assert((offset >= 1) && (offset <= MAX_DISTANCE));
+                opSaved = op;
                 if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) )   /* updates ip, op and anchor */
-                    return 0;  /* error */
+                    goto _dest_overflow;
         }   }
     }  /* while (ip < mflimit) */
 
+_last_literals:
     /* Encode Last Literals */
-    {   int lastRun = (int)(iend - anchor);
-        if ( (limit)
-          && (((char*)op - dst) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)dstCapacity))
-            return 0;  /* Check output limit */
-        if (lastRun >= (int)RUN_MASK) {
-            *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255;
-            *op++ = (BYTE) lastRun;
-        } else *op++ = (BYTE)(lastRun< oend)) {
+            if (limit == limitedOutput) return 0;  /* Check output limit */
+            /* adapt lastRunSize to fill 'dst' */
+            lastRunSize  = (size_t)(oend - op) - 1;
+            litLength = (lastRunSize + 255 - RUN_MASK) / 255;
+            lastRunSize -= litLength;
+        }
+        ip = anchor + lastRunSize;
+
+        if (lastRunSize >= RUN_MASK) {
+            size_t accumulator = lastRunSize - RUN_MASK;
+            *op++ = (RUN_MASK << ML_BITS);
+            for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
+            *op++ = (BYTE) accumulator;
+        } else {
+            *op++ = (BYTE)(lastRunSize << ML_BITS);
+        }
+        memcpy(op, anchor, lastRunSize);
+        op += lastRunSize;
     }
 
     /* End */
+    *srcSizePtr = (int) (((const char*)ip) - source);
     return (int) ((char*)op-dst);
+
+_dest_overflow:
+    if (limit == limitedDestSize) {
+        op = opSaved;  /* restore correct out pointer */
+        goto _last_literals;
+    }
+    return 0;
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 365ff79..c134fe3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -345,10 +345,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                   FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
 
                 DISPLAYLEVEL(5, " OK \n");
-            }
-            else
+            } else {
                 DISPLAYLEVEL(5, " \n");
-        }
+        }   }
 
         /* Test compression HC destSize */
         FUZ_DISPLAYTEST;
@@ -359,11 +358,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
             compressedBuffer[targetSize] = endCheck;
             ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
+            DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
+                            compressionLevel, ret, targetSize, srcSize, blockSize);
             LZ4_freeHC(ctx);
             FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
             FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
             FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
-            DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
+            DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
+                            compressionLevel, ret, targetSize, srcSize, blockSize);
             if (targetSize>0) {
                 /* check correctness */
                 U32 const crcBase = XXH32(block, srcSize, 0);
@@ -380,10 +382,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
                     FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
                 }
                 DISPLAYLEVEL(5, " OK \n");
-            }
-            else
+            } else {
                 DISPLAYLEVEL(5, " \n");
-        }
+        }   }
 
         /* Test compression HC */
         FUZ_DISPLAYTEST;
-- 
cgit v0.12


From e2900241e8c541edf3b8c18688caab238ad8d46f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 27 Dec 2017 13:02:40 +0100
Subject: object files in /tests

object files created from /tests are compiled with specific flags (`-g -DLZ4_DEBUG=1`)
which are not welcomed in release binary.
`lib/*.o` files created from /tests are now stored in /tests, to avoid unintentional mix.
---
 tests/Makefile | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index ce8c3fb..819ba43 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -78,21 +78,26 @@ lz4c unlz4 lz4cat: lz4
 lz4c32:   # create a 32-bits version for 32/64 interop tests
 	$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
 
-fullbench  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
+%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
+	$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
+
+fullbench  : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c
+$(LZ4DIR)/liblz4.a:
 	$(MAKE) -C $(LZ4DIR) liblz4.a
-	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
+
+fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
 	$(MAKE) -C $(LZ4DIR) liblz4
 	$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
 
-fuzzer  : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
+fuzzer  : lz4.o lz4hc.o xxhash.o fuzzer.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
+frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 datagen : $(PRGDIR)/datagen.c datagencli.c
-- 
cgit v0.12


From c8411ea880c4d2737537a5809e2e80ebf1307434 Mon Sep 17 00:00:00 2001
From: Eli Boyarski 
Date: Mon, 1 Jan 2018 14:15:31 +0200
Subject: Grammar: "to silent" -> "to silence"

---
 programs/lz4.1    | 2 +-
 programs/lz4.1.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/lz4.1 b/programs/lz4.1
index 39d78cd..7ce5da0 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -35,7 +35,7 @@ When writing scripts that need to decompress files, it is recommended to always
 \fBlz4 file\.lz4\fR will default to decompression (use \fB\-z\fR to force compression)
 .
 .IP "\(bu" 4
-\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them)
+\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silence them)
 .
 .IP "\(bu" 4
 If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\.
diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index c6b99bc..829c232 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -37,7 +37,7 @@ Differences are :
   * `lz4 file.lz4` will default to decompression (use `-z` to force compression)
   * `lz4` shows real-time notification statistics
      during compression or decompression of a single file
-     (use `-q` to silent them)
+     (use `-q` to silence them)
   * If no destination name is provided, result is sent to `stdout`
     _except if stdout is the console_.
   * If no destination name is provided, __and__ if `stdout` is the console,
-- 
cgit v0.12


From 329dcd4765d307aa1fad14d2e92c0f69ec45cec6 Mon Sep 17 00:00:00 2001
From: Eli Boyarski 
Date: Mon, 1 Jan 2018 15:34:16 +0200
Subject: Docs: describe behavior under nohup

---
 programs/lz4.1.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/programs/lz4.1.md b/programs/lz4.1.md
index c6b99bc..7bb14b0 100644
--- a/programs/lz4.1.md
+++ b/programs/lz4.1.md
@@ -45,6 +45,9 @@ Differences are :
   * As a consequence of previous rules, note the following example :
     `lz4 file | consumer` sends compressed data to `consumer` through `stdout`,
     hence it does _not_ create `file.lz4`.
+  * Another consequence of those rules is that to run `lz4` under `nohup`,
+    you should provide a destination file: `nohup lz4 file file.lz4`,
+    because `nohup` writes the specified command's output to a file.
 
 Default behaviors can be modified by opt-in commands, detailed below.
 
-- 
cgit v0.12


From 9474b706a511882c96c17306c02130b6718f2a09 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Jan 2018 11:27:24 -0800
Subject: [lz4io] Fix decompression file stat with --rm

---
 programs/lz4io.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 9bf4e93..2cf0c1c 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1073,23 +1073,27 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con
 
 static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename)
 {
+    stat_t statbuf;
+    int stat_result = 0;
     FILE* const foutput = LZ4IO_openDstFile(output_filename);
     if (foutput==NULL) return 1;   /* failure */
 
+    if ( strcmp(input_filename, stdinmark)
+      && UTIL_getFileStat(input_filename, &statbuf))
+        stat_result = 1;
+
     ress.dstFile = foutput;
     LZ4IO_decompressSrcFile(ress, input_filename, output_filename);
 
     fclose(foutput);
 
     /* Copy owner, file permissions and modification time */
-    {   stat_t statbuf;
-        if ( strcmp (input_filename, stdinmark)
-          && strcmp (output_filename, stdoutmark)
-          && strcmp (output_filename, nulmark)
-          && UTIL_getFileStat(input_filename, &statbuf) ) {
-            UTIL_setFileStat(output_filename, &statbuf);
-            /* should return value be read ? or is silent fail good enough ? */
-    }   }
+    if ( stat_result != 0
+      && strcmp (output_filename, stdoutmark)
+      && strcmp (output_filename, nulmark)) {
+        UTIL_setFileStat(output_filename, &statbuf);
+        /* should return value be read ? or is silent fail good enough ? */
+    }
 
     return 0;
 }
-- 
cgit v0.12


From 00eac87ddaae7cef6ae05e25ff417e092ceab509 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Jan 2018 11:32:04 -0800
Subject: [lz4io] Refuse to set file stat for non-regular files

---
 programs/util.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/programs/util.h b/programs/util.h
index 1382cab..fc7f63e 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -265,11 +265,17 @@ UTIL_STATIC void UTIL_waitForNextTick(void)
 #endif
 
 
+UTIL_STATIC int UTIL_isRegFile(const char* infilename);
+
+
 UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
 {
     int res = 0;
     struct utimbuf timebuf;
 
+    if (!UTIL_isRegFile(filename))
+        return -1;
+
     timebuf.actime = time(NULL);
     timebuf.modtime = statbuf->st_mtime;
     res += utime(filename, &timebuf);  /* set access and modification times */
-- 
cgit v0.12


From c2dd686e96c7c4ff6a84bd43b68d3c28369e1f95 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 5 Jan 2018 14:30:49 -0800
Subject: [lz4f] Skip memcpy() on empty dictionary

---
 lib/lz4frame.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index ebd1089..488ab75 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1631,7 +1631,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             if (dctx->tmpOutSize > 64 KB) copySize = 0;
             if (copySize > preserveSize) copySize = preserveSize;
 
-            memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
+            if (copySize > 0)
+                memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
 
             dctx->dict = dctx->tmpOutBuffer;
             dctx->dictSize = preserveSize + dctx->tmpOutStart;
@@ -1639,7 +1640,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
             const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;
             size_t const newDictSize = MIN(dctx->dictSize, 64 KB);
 
-            memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+            if (newDictSize > 0)
+                memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
 
             dctx->dict = dctx->tmpOutBuffer;
             dctx->dictSize = newDictSize;
-- 
cgit v0.12


From ebef34fe79bc6412894f6fb27bca60acf930a311 Mon Sep 17 00:00:00 2001
From: "W. Felix Handte" 
Date: Mon, 8 Jan 2018 14:46:22 -0500
Subject: Add Option to Make lz4frame_static.h Functions Visible in Shared
 Objects

In some contexts, *cough*like at facebook*cough*, dynamic linking is used in
contexts which aren't truly dynamic. That is, the guarantee is maintained that
a program will only ever execute against the library version it was compiled
to interact with.

For those situations, introduce a compile-time flag that overrides hiding
these unstable APIs in shared objects.
---
 lib/lz4frame_static.h | 43 +++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h
index 1899f8e..a59b94b 100644
--- a/lib/lz4frame_static.h
+++ b/lib/lz4frame_static.h
@@ -43,7 +43,15 @@ extern "C" {
 /* lz4frame_static.h should be used solely in the context of static linking.
  * It contains definitions which are not stable and may change in the future.
  * Never use it in the context of DLL linking.
+ *
+ * Defining LZ4F_PUBLISH_STATIC_FUNCTIONS allows one to override this. Use at
+ * your own risk.
  */
+#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
+#define LZ4FLIB_STATIC_API LZ4FLIB_API
+#else
+#define LZ4FLIB_STATIC_API
+#endif
 
 
 /* ---   Dependency   --- */
@@ -79,7 +87,7 @@ extern "C" {
 /* enum list is exposed, to handle specific errors */
 typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;
 
-LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
+LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
 
 
 
@@ -93,8 +101,8 @@ typedef struct LZ4F_CDict_s LZ4F_CDict;
  *  LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
  *  LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
  * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */
-LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
-void        LZ4F_freeCDict(LZ4F_CDict* CDict);
+LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
+LZ4FLIB_STATIC_API void        LZ4F_freeCDict(LZ4F_CDict* CDict);
 
 
 /*! LZ4_compressFrame_usingCDict() :
@@ -106,10 +114,11 @@ void        LZ4F_freeCDict(LZ4F_CDict* CDict);
  *  but it's not recommended, as it's the only way to provide dictID in the frame header.
  * @return : number of bytes written into dstBuffer.
  *           or an error code if it fails (can be tested using LZ4F_isError()) */
-size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
-                               const void* src, size_t srcSize,
-                               const LZ4F_CDict* cdict,
-                               const LZ4F_preferences_t* preferencesPtr);
+LZ4FLIB_STATIC_API size_t LZ4F_compressFrame_usingCDict(
+    void* dst, size_t dstCapacity,
+    const void* src, size_t srcSize,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* preferencesPtr);
 
 
 /*! LZ4F_compressBegin_usingCDict() :
@@ -119,21 +128,23 @@ size_t LZ4F_compressFrame_usingCDict(void* dst, size_t dstCapacity,
  *  however, it's the only way to provide dictID in the frame header.
  * @return : number of bytes written into dstBuffer for the header,
  *           or an error code (which can be tested using LZ4F_isError()) */
-size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx,
-                                     void* dstBuffer, size_t dstCapacity,
-                                     const LZ4F_CDict* cdict,
-                                     const LZ4F_preferences_t* prefsPtr);
+LZ4FLIB_STATIC_API size_t LZ4F_compressBegin_usingCDict(
+    LZ4F_cctx* cctx,
+    void* dstBuffer, size_t dstCapacity,
+    const LZ4F_CDict* cdict,
+    const LZ4F_preferences_t* prefsPtr);
 
 
 /*! LZ4F_decompress_usingDict() :
  *  Same as LZ4F_decompress(), using a predefined dictionary.
  *  Dictionary is used "in place", without any preprocessing.
  *  It must remain accessible throughout the entire frame decoding. */
-size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr,
-                       void* dstBuffer, size_t* dstSizePtr,
-                       const void* srcBuffer, size_t* srcSizePtr,
-                       const void* dict, size_t dictSize,
-                       const LZ4F_decompressOptions_t* decompressOptionsPtr);
+LZ4FLIB_STATIC_API size_t LZ4F_decompress_usingDict(
+    LZ4F_dctx* dctxPtr,
+    void* dstBuffer, size_t* dstSizePtr,
+    const void* srcBuffer, size_t* srcSizePtr,
+    const void* dict, size_t dictSize,
+    const LZ4F_decompressOptions_t* decompressOptionsPtr);
 
 
 #if defined (__cplusplus)
-- 
cgit v0.12


From fb5e98a334d5538f4ffa70f5f51b8286531fe1ae Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 Jan 2018 14:36:10 -0800
Subject: updated NEWS for v1.8.1

---
 NEWS | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 89398b9..f9c503f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,9 +1,12 @@
 v1.8.1
-perf : faster and stronger ultra modes (level 11 and 12)
+perf : faster and stronger ultra modes (levels 10+)
 perf : slightly faster compression and decompression speed
 perf : fix bad degenerative case, reported by @c-morgenstern
 fix : decompression failed when using a combination of extDict + low memory address (#397), reported and fixed by Julian Scheid (@jscheid)
 cli : support for dictionary compression (`-D`), by Felix Handte @felixhandte
+cli : fix : `lz4 -d --rm` preserves timestamp (#441)
+cli : fix : do not modify /dev/null permission as root, by @aliceatlas
+api : `_destSize()` variant supported for all compression levels
 build  : `make` and `make test` compatible with `-jX`, reported by @mwgamera
 build  : can control LZ4LIB_VISIBILITY macro, by @mikir
 install: fix man page directory (#387), reported by Stuart Cardall (@itoffshore)
-- 
cgit v0.12


From c423dc21bde53869739d85c08071803662278f5b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 13 Jan 2018 13:16:31 -0800
Subject: updated LZ4F_decompress() documentation

---
 lib/lz4frame.h | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 88a6513..675aeeb 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -347,27 +347,32 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
 
 /*! LZ4F_decompress() :
  *  Call this function repetitively to regenerate compressed data from `srcBuffer`.
- *  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+ *  The function will read up to *srcSizePtr bytes from srcBuffer,
+ *  and decompress data into dstBuffer, of capacity *dstSizePtr.
  *
- *  The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+ *  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+ *  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
  *
- *  The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
- *  Number of bytes consumed can be < number of bytes provided.
- *  It typically happens when dstBuffer is not large enough to contain all decoded data.
+ *  The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
  *  Unconsumed source data must be presented again in subsequent invocations.
  *
- * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- * `dstBuffer` itself can be changed at will between each consecutive function invocation.
+ * `dstBuffer` can freely change between each consecutive function invocation.
+ * `dstBuffer` content will be overwritten.
  *
  * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
  *  Schematically, it's the size of the current (or remaining) compressed block + header of next block.
  *  Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
  *  This is just a hint though, it's always possible to provide any srcSize.
+ *
  *  When a frame is fully decoded, @return will be 0 (no more data expected).
+ *  When provided with more bytes than necessary to decode a frame,
+ *  LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0.
+ *
  *  If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+ *  After a decompression error, the `dctx` context is not resumable.
+ *  Use LZ4F_resetDecompressionContext() to return to clean state.
  *
  *  After a frame is fully decoded, dctx can be used again to decompress another frame.
- *  After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  */
 LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    void* dstBuffer, size_t* dstSizePtr,
@@ -375,11 +380,11 @@ LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx,
                                    const LZ4F_decompressOptions_t* dOptPtr);
 
 
-/*! LZ4F_resetDecompressionContext() : v1.8.0
+/*! LZ4F_resetDecompressionContext() : added in v1.8.0
  *  In case of an error, the context is left in "undefined" state.
  *  In which case, it's necessary to reset it, before re-using it.
- *  This method can also be used to abruptly stop an unfinished decompression,
- *  and start a new one using the same context. */
+ *  This method can also be used to abruptly stop any unfinished decompression,
+ *  and start a new one using same context resources. */
 LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
 
 
-- 
cgit v0.12


From 47bf1a9f010d548fa58c7716678e7a11b356b120 Mon Sep 17 00:00:00 2001
From: Po-Chuan Hsieh 
Date: Sun, 14 Jan 2018 06:38:03 +0800
Subject: Fix lz4 version

---
 lib/lz4.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4.h b/lib/lz4.h
index 3cf6c8d..a06b8a4 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -93,7 +93,7 @@ extern "C" {
 /*------   Version   ------*/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ4_VERSION_MINOR    8    /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
 
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 
-- 
cgit v0.12


From ac38ffa9afed07f8bcd291b35b89adf35f3ba28f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 13 Jan 2018 18:46:13 -0800
Subject: updated manuals to v1.8.1

---
 programs/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index ed1a779..a51bd4b 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -88,8 +88,8 @@ lz4c32: CFLAGS += -m32
 lz4c32 : $(SRCFILES)
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-lz4.1: lz4.1.md
-	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+lz4.1: lz4.1.md $(LIBVER_SRC)
+	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 
 man: lz4.1
 
-- 
cgit v0.12


From e8ee6e5965249f52a4bc14c140349803728113f9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 13 Jan 2018 18:51:33 -0800
Subject: nicer console message for `make clean`

---
 Makefile                 |  2 +-
 doc/lz4_manual.html      |  4 ++--
 doc/lz4frame_manual.html | 29 +++++++++++++++++------------
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 76fc9a8..0aa1b9d 100644
--- a/Makefile
+++ b/Makefile
@@ -82,7 +82,7 @@ clean:
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(EXDIR) $@ > $(VOID)
-	@$(MAKE) -C contrib/gen_manual $@
+	@$(MAKE) -C contrib/gen_manual $@ > $(VOID)
 	@$(RM) lz4$(EXT)
 	@echo Cleaning completed
 
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 4c9c7de..6b7935d 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.8.0 Manual
+1.8.1 Manual
 
 
-
1.8.0 Manual

+
1.8.1 Manual

 

 
Contents

 
    
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 7529f6e..590c632 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -1,10 +1,10 @@
 
 
 
-1.8.0 Manual
+1.8.1 Manual
 
 
-
    1.8.0 Manual
    
+
    1.8.1 Manual
    
 
    
 
    Contents
    
 
      
@@ -242,35 +242,40 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
                                    const void* srcBuffer, size_t* srcSizePtr,
                                    const LZ4F_decompressOptions_t* dOptPtr);
 
        Call this function repetitively to regenerate compressed data from `srcBuffer`.
-  The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr.
+  The function will read up to *srcSizePtr bytes from srcBuffer,
+  and decompress data into dstBuffer, of capacity *dstSizePtr.
 
-  The number of bytes regenerated into dstBuffer is provided within *dstSizePtr (necessarily <= original value).
+  The number of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
+  The number of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value).
 
-  The number of bytes consumed from srcBuffer is provided within *srcSizePtr (necessarily <= original value).
-  Number of bytes consumed can be < number of bytes provided.
-  It typically happens when dstBuffer is not large enough to contain all decoded data.
+  The function does not necessarily read all input bytes, so always check value in *srcSizePtr.
   Unconsumed source data must be presented again in subsequent invocations.
 
- `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten.
- `dstBuffer` itself can be changed at will between each consecutive function invocation.
+ `dstBuffer` can freely change between each consecutive function invocation.
+ `dstBuffer` content will be overwritten.
 
  @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call.
   Schematically, it's the size of the current (or remaining) compressed block + header of next block.
   Respecting the hint provides some small speed benefit, because it skips intermediate buffers.
   This is just a hint though, it's always possible to provide any srcSize.
+
   When a frame is fully decoded, @return will be 0 (no more data expected).
+  When provided with more bytes than necessary to decode a frame,
+  LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0.
+
   If decompression failed, @return is an error code, which can be tested using LZ4F_isError().
+  After a decompression error, the `dctx` context is not resumable.
+  Use LZ4F_resetDecompressionContext() to return to clean state.
 
   After a frame is fully decoded, dctx can be used again to decompress another frame.
-  After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state.
  
 


 
 
void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx);   /* always successful */
 
  In case of an error, the context is left in "undefined" state.
   In which case, it's necessary to reset it, before re-using it.
-  This method can also be used to abruptly stop an unfinished decompression,
-  and start a new one using the same context. 
+  This method can also be used to abruptly stop any unfinished decompression,
+  and start a new one using same context resources. 
 


 
 
-- 
cgit v0.12


From 52a76ff682042a310528c1c1bbe0d75c28131449 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 13 Jan 2018 19:06:33 -0800
Subject: update man page

---
 programs/lz4.1 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/programs/lz4.1 b/programs/lz4.1
index 7ce5da0..e0f6a81 100644
--- a/programs/lz4.1
+++ b/programs/lz4.1
@@ -1,5 +1,5 @@
 .
-.TH "LZ4" "1" "July 2017" "lz4 1.8.0" "User Commands"
+.TH "LZ4" "1" "2018-01-13" "lz4 1.8.1" "User Commands"
 .
 .SH "NAME"
 \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files
@@ -46,6 +46,9 @@ If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \f
 .IP "\(bu" 4
 As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create \fBfile\.lz4\fR\.
 .
+.IP "\(bu" 4
+Another consequence of those rules is that to run \fBlz4\fR under \fBnohup\fR, you should provide a destination file: \fBnohup lz4 file file\.lz4\fR, because \fBnohup\fR writes the specified command\'s output to a file\.
+.
 .IP "" 0
 .
 .P
-- 
cgit v0.12