From 65a542579b4849a6b064c8c8b3baf16e31119294 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 22 Nov 2016 11:14:11 +0100 Subject: updated README.md --- lib/README.md | 2 +- lib/dll/example/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/README.md b/lib/README.md index 18793d4..b40442c 100644 --- a/lib/README.md +++ b/lib/README.md @@ -47,7 +47,7 @@ The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.lib". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll ``` diff --git a/lib/dll/example/README.md b/lib/dll/example/README.md index 4908663..223e473 100644 --- a/lib/dll/example/README.md +++ b/lib/dll/example/README.md @@ -39,7 +39,7 @@ The header files from `include\` and the dynamic library `dll\liblz4.dll` are required to compile a project using gcc/MinGW. The dynamic library has to be added to linking options. It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be compiled with "liblz4.dll". For example: +file it should be linked with `dll\liblz4.dll`. For example: ``` gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\liblz4.dll ``` -- cgit v0.12 From de0cf5de640eb56775f44c0f4c7c7a065e00447c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 22 Nov 2016 11:20:02 +0100 Subject: fixed LZ4IO_compressMultipleFilenames --- programs/lz4io.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index bc2ba95..5e45e72 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -593,10 +593,12 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const size_t suffixSize = strlen(suffix); cRess_t const ress = LZ4IO_createCResources(); + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + /* loop on each file */ for (i=0; i Date: Sun, 27 Nov 2016 23:40:53 +0900 Subject: Add "man" target to Makefile - `make man` makes manpage lz4.1 from lz4.1.md - It needs `gem` and `ronn` which is same as xxhash.1.md is required. --- programs/Makefile | 27 +++++++ programs/lz4.1.md | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 programs/lz4.1.md diff --git a/programs/Makefile b/programs/Makefile index 504416f..ffbcbf1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,6 +27,17 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## +# Version numbers +LIBVER_SRC := ../lib/lz4.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + DESTDIR ?= PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin @@ -42,6 +53,10 @@ DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) +LZ4_VERSION=$(LIBVER) +MD2ROFF =ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" + # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -80,6 +95,18 @@ clean: @echo Cleaning completed +lz4.1: lz4.1.md + cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: lz4.1 + +clean-man: + rm lz4.1 + +preview-man: clean-man man + man ./lz4.1 + + #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD, Hurd and #FreeBSD targets diff --git a/programs/lz4.1.md b/programs/lz4.1.md new file mode 100644 index 0000000..64ce428 --- /dev/null +++ b/programs/lz4.1.md @@ -0,0 +1,206 @@ +lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files +================================================================ + +SYNOPSIS +-------- + +`lz4` [*OPTIONS*] [-|INPUT-FILE] + +`unlz4` is equivalent to `lz4 -d` + +`lz4cat` is equivalent to `lz4 -dcfm` + +When writing scripts that need to decompress files, +it is recommended to always use the name `lz4` with appropriate arguments +(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. + + +DESCRIPTION +----------- + +`lz4` is an extremely fast lossless compression algorithm, +based on **byte-aligned LZ77** family of compression scheme. +`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with +multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the `.lz4` format. + +### Difference between lz4 and gzip + +`lz4` supports a command line syntax similar +_but not identical_ to `gzip(1)`. +Differences are : + + * `lz4` preserves original files + * `lz4` compresses a single file by default (use `-m` for multiple files) + * `lz4 file1 file2` means : compress file1 _into_ file2 + * When no destination name is provided, compressed file name receives + a `.lz4` suffix + * When no destination name is provided, if `stdout` is _not_ the console, + it becomes the output (like a silent `-c`). + Therefore `lz4 file > /dev/null` will not create `file.lz4` + * `lz4 file` shows real-time statistics during compression + (use `-q` to silent them) + +Default behaviors can be modified by opt-in commands, described below. +`lz4 --quiet --multiple` more closely mimics `gzip` behavior. + +### Concatenation of .lz4 files + +It is possible to concatenate `.lz4` files as is. +`lz4` will decompress such files as if they were a single `.lz4` file. +For example: + lz4 file1 > foo.lz4 + lz4 file2 >> foo.lz4 + +then + lz4cat foo.lz4 + +is equivalent to : + cat file1 file2 + + +OPTIONS +------- + +### Short commands concatenation + +In some cases, some options can be expressed using short command `-x` +or long command `--long-word`. +Short commands can be concatenated together. +For example, `-d -c` is equivalent to `-dc`. +Long commands cannot be concatenated. +They must be clearly separated by a space. + +### Multiple commands + +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. + +### Operation mode + +* `-z` `--compress`: + Compress. + This is the default operation mode when no operation mode option is + specified, no other operation mode is implied from the command name + (for example, `unlz4` implies `--decompress`), + nor from the input file name + (for example, a file extension `.lz4` implies `--decompress` by default). + `-z` can also be used to force compression of an already compressed + `.lz4` file. + +* `-d` `--decompress` `--uncompress`: + Decompress. + `--decompress` is also the default operation when the input filename has an + `.lz4` extension. + +* `-t` `--test`: + Test the integrity of compressed `.lz4` files. + The decompressed data is discarded. + No files are created nor removed. + +* `-b#`: + Benchmark mode, using `#` compression level. + +### Operation modifiers + +* `-#`: + Compression level, with # being any value from 1 to 16. + Higher values trade compression speed for compression ratio. + Values above 16 are considered the same as 16. + Recommended values are 1 for fast compression (default), + and 9 for high compression. + Speed/compression trade-off will vary depending on data to compress. + Decompression speed remains fast at all settings. + +* `-f` `--[no-]force`: + This option has several effects: + + If the target file already exists, overwrite it without prompting. + + When used with `--decompress` and `lz4` cannot recognize the type of + the source file, copy the source file as is to standard output. + This allows `lz4cat --force` to be used like `cat (1)` for files + that have not been compressed with `lz4`. + +* `-c` `--stdout` `--to-stdout`: + Force write to standard output, even if it is the console. + +* `-m` `--multiple`: + Multiple file names. + By default, the second filename is used as the destination filename + for the compressed file. + With `-m`, it is possible to specify any number of input filenames. + Each of them will be compressed independently, and the resulting name of + each compressed file will be `filename.lz4`. + +* `-B#`: + Block size \[4-7\](default : 7)
+ `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB + +* `-BD`: + Block dependency (improves compression ratio on small blocks) + +* `--[no-]frame-crc`: + Select frame checksum (default:enabled) + +* `--[no-]content-size`: + Header includes original size (default:not present)
+ Note : this option can only be activated when the original size can be + determined, hence for a file. It won't work with unknown source size, + such as stdin or pipe. + +* `--[no-]sparse`: + Sparse mode support (default:enabled on file, disabled on stdout) + +* `-l`: + Use Legacy format (typically used for Linux Kernel compression)
+ Note : `-l` is not compatible with `-m` (`--multiple`) + +### Other options + +* `-v` `--verbose`: + Verbose mode + +* `-q` `--quiet`: + Suppress warnings and real-time statistics; specify twice to suppress + errors too + +* `-h` `-H` `--help`: + Display help/long help and exit + +* `-V` `--version`: + Display Version number and exit + +* `-k` `--keep`: + Don't delete source file. + This is default behavior anyway, so this option is just for compatibility + with `gzip(1)` / `xz(1)`. + + +### Benchmark mode + +* `-b#`: + Benchmark file(s), using # compression level + +* `-e#`: + Benchmark multiple compression levels, from b# to e# (included) + +* `-i#`: + Minimum evaluation in seconds \[1-9\] (default : 3) + +* `-r`: + Operate recursively on directories + + +BUGS +---- + +Report bugs at: https://github.com/lz4/lz4/issues + + +AUTHOR +------ + +Yann Collet -- cgit v0.12 From 009f06e01d424861048727ab5d9486f7a60a3a6e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 27 Nov 2016 23:23:40 -0800 Subject: minor man page update --- programs/lz4.1 | 389 ++++++++++++++++++++++-------------------------------- programs/lz4.1.md | 10 +- 2 files changed, 168 insertions(+), 231 deletions(-) diff --git a/programs/lz4.1 b/programs/lz4.1 index 2c94033..d73a7ed 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,251 +1,184 @@ -\." -\." lz4.1: This is a manual page for 'lz4' program. This file is part of the -\." lz4 project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lz4 "1" "2015-03-21" "lz4" "User Commands" -.SH NAME -\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] -.PP -.B unlz4 -is equivalent to -.BR "lz4 \-d" -.br -.B lz4cat -is equivalent to -.BR "lz4 \-dcfm" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lz4 -with appropriate arguments -.RB ( "lz4 \-d" -or -.BR "lz4 \-dc" ) -instead of the names -.B unlz4 -and -.BR lz4cat . - - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .lz4 -format. - -.B lz4 -supports a command line syntax similar \fIbut not identical\fR to -.BR gzip (1). -Differences are : - \fBlz4\fR preserves original files - \fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files) - \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 - When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix - When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR) - Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR - \fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them) - -Default behaviors can be modified by opt-in commands, described below. -\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .lz4 files" -It is possible to concatenate -.B .lz4 -files as is. -.B lz4 -will decompress such files as if they were a single -.B .lz4 -file. For example: - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - then - lz4cat foo.lz4 - is equivalent to : - cat file1 file2 - -.PP - -.SH OPTIONS +. +.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands" +. +.SH "NAME" +\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files +. +.SH "SYNOPSIS" +\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR +. +.P +\fBunlz4\fR is equivalent to \fBlz4 \-d\fR +. +.P +\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR +. +.P +When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. +. +.SH "DESCRIPTION" +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +. +.SS "Difference between lz4 and gzip" +\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : +. +.IP "\(bu" 4 +\fBlz4\fR preserves original files +. +.IP "\(bu" 4 +\fBlz4\fR compresses a single file by default (use \fB\-m\fR for multiple files) +. +.IP "\(bu" 4 +\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 +. +.IP "\(bu" 4 +When no destination name is provided, compressed file name receives a \fB\.lz4\fR suffix +. +.IP "\(bu" 4 +When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB\-c\fR)\. Therefore \fBlz4 file > /dev/null\fR will not create \fBfile\.lz4\fR +. +.IP "\(bu" 4 +\fBlz4 file\fR shows real\-time statistics during compression (use \fB\-q\fR to silent them) +. +.IP "" 0 +. +.P +Default behaviors can be modified by opt\-in commands, described below\. \fBlz4 \-m\fR more closely follows \fBgzip\fR command line syntax, supporting compression of multiple files into \fB\.lz4\fR, the last remaining difference being that source files are preserved by default\. Since it\'s also possible to erase source file using opt\-in command \fB\-\-rm\fR, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.SS "Concatenation of \.lz4 files" +It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 +. +.P +then lz4cat foo\.lz4 +. +.P +is equivalent to : cat file1 file2 +. +.SH "OPTIONS" . .SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. +In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. +. .SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. +When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. . .SS "Operation mode" +. +.TP +\fB\-z\fR \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlz4 -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .lz4 -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .lz4 -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .lz4 -extension. -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .lz4 -files. -The decompressed data is discarded. -No files are created nor removed. - -.TP -.BR \-b# -Benchmark mode, using # compression level. - +\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR +Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. +. +.TP +\fB\-t\fR \fB\-\-test\fR +Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. +. +.TP +\fB\-b#\fR +Benchmark mode, using \fB#\fR compression level\. . .SS "Operation modifiers" +. +.TP +\fB\-#\fR +Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. +. +.TP +\fB\-f\fR \fB\-\-[no\-]force\fR +This option has several effects: +. +.IP +If the target file already exists, overwrite it without prompting\. +. +.IP +When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. +. +.TP +\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR +Force write to standard output, even if it is the console\. +. +.TP +\fB\-m\fR \fB\-\-multiple\fR +Multiple file names\. By default, the second filename is used as the destination filename for the compressed file\. With \fB\-m\fR, it is possible to specify any number of input filenames\. Each of them will be compressed independently, and the resulting name of each compressed file will be \fBfilename\.lz4\fR\. This mode also reduces notification level\. +. +.TP +\fB\-B#\fR +Block size [4\-7](default : 7) +. +.br +\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB +. +.TP +\fB\-BD\fR +Block dependency (improves compression ratio on small blocks) +. +.TP +\fB\-\-[no\-]frame\-crc\fR +Select frame checksum (default:enabled) +. +.TP +\fB\-\-[no\-]content\-size\fR +Header includes original size (default:not present) +. +.br +Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. +. .TP -.B \-# - compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - -.TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lz4 -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lz4cat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lz4 . -.RE - -.TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - -.TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, it is possible to specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.lz4 -. - -.TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB -.TP -.B \-BD - block dependency (improves compression ratio on small blocks) -.TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) -.TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. -.TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) -.TP -.B \-l - use Legacy format (typically used for Linux Kernel compression) - note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR) +\fB\-\-[no\-]sparse\fR +Sparse mode support (default:enabled on file, disabled on stdout) +. +.TP +\fB\-l\fR +Use Legacy format (typically used for Linux Kernel compression) +. +.br +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) . .SS "Other options" +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-v\fR \fB\-\-verbose\fR +Verbose mode +. .TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too +\fB\-q\fR \fB\-\-quiet\fR +Suppress warnings and real\-time statistics; specify twice to suppress errors too +. .TP -.B \-h/\-H ", " --help - display help/long help and exit +\fB\-h\fR \fB\-H\fR \fB\-\-help\fR +Display help/long help and exit +. .TP -.BR \-V ", " \--version - display Version number and exit +\fB\-V\fR \fB\-\-version\fR +Display Version number and exit +. .TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. - +\fB\-k\fR \fB\-\-keep\fR +Don\'t delete source file\. This is default behavior anyway, so this option is just for compatibility with \fBgzip(1)\fR / \fBxz(1)\fR\. . .SS "Benchmark mode" +. .TP -.B \-b# - benchmark file(s), using # compression level +\fB\-b#\fR +Benchmark file(s), using # compression level +. .TP -.B \-e# - benchmark multiple compression levels, from b# to e# (included) +\fB\-e#\fR +Benchmark multiple compression levels, from b# to e# (included) +. .TP -.B \-i# - minimum evaluation in seconds [1-9] (default : 3) +\fB\-i#\fR +Minimum evaluation in seconds [1\-9] (default : 3) +. .TP -.B \-r - operate recursively on directories - - -.SH BUGS -Report bugs at: https://github.com/Cyan4973/lz4/issues - -.SH AUTHOR +\fB\-r\fR +Operate recursively on directories +. +.SH "BUGS" +Report bugs at: https://github\.com/lz4/lz4/issues +. +.SH "AUTHOR" Yann Collet diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 64ce428..f10680d 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -28,8 +28,7 @@ The native file format is the `.lz4` format. ### Difference between lz4 and gzip -`lz4` supports a command line syntax similar -_but not identical_ to `gzip(1)`. +`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. Differences are : * `lz4` preserves original files @@ -44,7 +43,11 @@ Differences are : (use `-q` to silent them) Default behaviors can be modified by opt-in commands, described below. -`lz4 --quiet --multiple` more closely mimics `gzip` behavior. +`lz4 -m` more closely follows `gzip` command line syntax, +supporting compression of multiple files into `.lz4`, +the last remaining difference being that source files are preserved by default. +Since it's also possible to erase source file using opt-in command `--rm`, +`lz4 -m --rm` behaves the same as `gzip`. ### Concatenation of .lz4 files @@ -134,6 +137,7 @@ only the latest one will be applied. With `-m`, it is possible to specify any number of input filenames. Each of them will be compressed independently, and the resulting name of each compressed file will be `filename.lz4`. + This mode also reduces notification level. * `-B#`: Block size \[4-7\](default : 7)
-- cgit v0.12 From 6b90cfbeab05fb2a74b613352a038330b83da61a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 03:43:01 -0800 Subject: Updated man page --- programs/lz4.1 | 42 ++++++++++++++++++++++++++-------- programs/lz4.1.md | 68 +++++++++++++++++++++++++++++++------------------------ 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/programs/lz4.1 b/programs/lz4.1 index d73a7ed..48c988a 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -26,24 +26,38 @@ When writing scripts that need to decompress files, it is recommended to always \fBlz4\fR preserves original files . .IP "\(bu" 4 -\fBlz4\fR compresses a single file by default (use \fB\-m\fR for multiple files) +\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) . .IP "\(bu" 4 \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 . .IP "\(bu" 4 -When no destination name is provided, compressed file name receives a \fB\.lz4\fR suffix +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) . .IP "\(bu" 4 -When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB\-c\fR)\. Therefore \fBlz4 file > /dev/null\fR will not create \fBfile\.lz4\fR +If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. . .IP "\(bu" 4 -\fBlz4 file\fR shows real\-time statistics during compression (use \fB\-q\fR to silent them) +If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\. . .IP "" 0 . .P -Default behaviors can be modified by opt\-in commands, described below\. \fBlz4 \-m\fR more closely follows \fBgzip\fR command line syntax, supporting compression of multiple files into \fB\.lz4\fR, the last remaining difference being that source files are preserved by default\. Since it\'s also possible to erase source file using opt\-in command \fB\-\-rm\fR, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +Default behaviors can be modified by opt\-in commands, detailed below\. +. +.IP "\(bu" 4 +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\. +. +.IP "\(bu" 4 +It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.IP "" 0 . .SS "Concatenation of \.lz4 files" It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 @@ -102,7 +116,11 @@ Force write to standard output, even if it is the console\. . .TP \fB\-m\fR \fB\-\-multiple\fR -Multiple file names\. By default, the second filename is used as the destination filename for the compressed file\. With \fB\-m\fR, it is possible to specify any number of input filenames\. Each of them will be compressed independently, and the resulting name of each compressed file will be \fBfilename\.lz4\fR\. This mode also reduces notification level\. +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. +. +.TP +\fB\-r\fR +operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. . .TP \fB\-B#\fR @@ -113,7 +131,7 @@ Block size [4\-7](default : 7) . .TP \fB\-BD\fR -Block dependency (improves compression ratio on small blocks) +Block Dependency (improves compression ratio on small blocks) . .TP \fB\-\-[no\-]frame\-crc\fR @@ -132,10 +150,10 @@ Sparse mode support (default:enabled on file, disabled on stdout) . .TP \fB\-l\fR -Use Legacy format (typically used for Linux Kernel compression) +Use Legacy format (typically for Linux Kernel compression) . .br -Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR . .SS "Other options" . @@ -157,7 +175,11 @@ Display Version number and exit . .TP \fB\-k\fR \fB\-\-keep\fR -Don\'t delete source file\. This is default behavior anyway, so this option is just for compatibility with \fBgzip(1)\fR / \fBxz(1)\fR\. +Preserve source files (default behavior) +. +.TP +\fB\-\-rm\fR +Delete source files on successful compression or decompression . .SS "Benchmark mode" . diff --git a/programs/lz4.1.md b/programs/lz4.1.md index f10680d..71ad326 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -32,22 +32,29 @@ The native file format is the `.lz4` format. Differences are : * `lz4` preserves original files - * `lz4` compresses a single file by default (use `-m` for multiple files) + * `lz4` compresses a single file by default (see `-m` for multiple files) * `lz4 file1 file2` means : compress file1 _into_ file2 - * When no destination name is provided, compressed file name receives - a `.lz4` suffix - * When no destination name is provided, if `stdout` is _not_ the console, - it becomes the output (like a silent `-c`). - Therefore `lz4 file > /dev/null` will not create `file.lz4` - * `lz4 file` shows real-time statistics during compression - (use `-q` to silent them) - -Default behaviors can be modified by opt-in commands, described below. -`lz4 -m` more closely follows `gzip` command line syntax, -supporting compression of multiple files into `.lz4`, -the last remaining difference being that source files are preserved by default. -Since it's also possible to erase source file using opt-in command `--rm`, -`lz4 -m --rm` behaves the same as `gzip`. + * `lz4` shows real-time notification statistics + during compression or decompression of a single file + (use `-q` to silent them) + * If no destination name is provided, result is sent to `stdout` + _except if stdout is the console_. + * If no destination name is provided, __and__ if `stdout` is the console, + `file` is compressed into `file.lz4`. + * As a consequence of previous rules, note the following example : + `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, + hence it does _not_ create any `file.lz4`. + +Default behaviors can be modified by opt-in commands, detailed below. + + * `lz4 -m` makes it possible to provide multiple input filenames, + which will be compressed into files using suffix `.lz4`. + Progress notifications are also disabled by default. + This mode has a behavior which more closely mimics `gzip` command line, + with the main difference being that source files are preserved by default. + * It's possible to opt-in to erase source files + on successful compression or decompression, using `--rm` command. + * Consequently, `lz4 -m --rm` behaves the same as `gzip`. ### Concatenation of .lz4 files @@ -131,20 +138,22 @@ only the latest one will be applied. Force write to standard output, even if it is the console. * `-m` `--multiple`: - Multiple file names. - By default, the second filename is used as the destination filename - for the compressed file. - With `-m`, it is possible to specify any number of input filenames. - Each of them will be compressed independently, and the resulting name of - each compressed file will be `filename.lz4`. + Multiple input files. + Compressed file names will be appended a `.lz4` suffix. This mode also reduces notification level. + `lz4 -m` has a behavior equivalent to `gzip -k` + (it preserves source files by default). + +* `-r` : + operate recursively on directories. + This mode also sets `-m` (multiple input files). * `-B#`: Block size \[4-7\](default : 7)
`-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB * `-BD`: - Block dependency (improves compression ratio on small blocks) + Block Dependency (improves compression ratio on small blocks) * `--[no-]frame-crc`: Select frame checksum (default:enabled) @@ -159,8 +168,8 @@ only the latest one will be applied. Sparse mode support (default:enabled on file, disabled on stdout) * `-l`: - Use Legacy format (typically used for Linux Kernel compression)
- Note : `-l` is not compatible with `-m` (`--multiple`) + Use Legacy format (typically for Linux Kernel compression)
+ Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` ### Other options @@ -168,8 +177,8 @@ only the latest one will be applied. Verbose mode * `-q` `--quiet`: - Suppress warnings and real-time statistics; specify twice to suppress - errors too + Suppress warnings and real-time statistics; + specify twice to suppress errors too * `-h` `-H` `--help`: Display help/long help and exit @@ -178,9 +187,10 @@ only the latest one will be applied. Display Version number and exit * `-k` `--keep`: - Don't delete source file. - This is default behavior anyway, so this option is just for compatibility - with `gzip(1)` / `xz(1)`. + Preserve source files (default behavior) + +* `--rm` : + Delete source files on successful compression or decompression ### Benchmark mode -- cgit v0.12 From a9a8dbf7c10125f6024943df2e84108de49ce8e8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 04:01:02 -0800 Subject: fixed minor display error when using -r --- NEWS | 3 +++ lib/lz4.h | 2 +- programs/lz4.1.md | 3 --- programs/lz4cli.c | 45 ++++++++++++++++++++++++--------------------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index f6874ce..56a2875 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v1.7.5 +cli : fix minor notification when using -r recursive mode + v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/lib/lz4.h b/lib/lz4.h index 7420ad8..0aae19c 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -85,7 +85,7 @@ extern "C" { /*========== Version =========== */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 71ad326..83a08b6 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -204,9 +204,6 @@ only the latest one will be applied. * `-i#`: Minimum evaluation in seconds \[1-9\] (default : 3) -* `-r`: - Operate recursively on directories - BUGS ---- diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 71378aa..a452d1c 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -129,19 +129,20 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output *****************************/ static int usage(const char* exeName) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "Usage : \n"); + DISPLAY( " %s [arg] [input] [output] \n", exeName); DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); + DISPLAY( "input : a filename \n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); + DISPLAY( "Arguments : \n"); DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); + DISPLAY( " -z : force compression \n"); DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -h/-H : display help/long help and exit \n"); return 0; } @@ -151,33 +152,33 @@ static int usage_advanced(const char* exeName) usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -V : display Version number and exit \n"); + DISPLAY( " -v : verbose mode \n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); #ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); + DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); + DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [4-7] (default: 7)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); + DISPLAY( "Legacy arguments : \n"); + DISPLAY( " -c0 : fast compression \n"); + DISPLAY( " -c1 : high compression \n"); + DISPLAY( " -hc : high compression \n"); DISPLAY( " -y : overwrite output without prompting \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ EXTENDED_HELP; @@ -548,7 +549,7 @@ int main(int argc, const char** argv) } /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) { + while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ size_t const inSize = strlen(input_filename); @@ -583,6 +584,8 @@ int main(int argc, const char** argv) break; } + if (!output_filename) output_filename = "*\\dummy^!//"; + /* Check if output is defined as console; trigger an error in this case */ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); -- cgit v0.12 From 9a6f7dfc5b1e9ce882944c45de895cf9c21d1b7c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Nov 2016 04:05:37 -0800 Subject: updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 56a2875..b879469 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v1.7.5 cli : fix minor notification when using -r recursive mode +doc : markdown version of man page, by Takayuki Matsuoka v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From 5eb547a6080b8e63b5e522b448e31052fbc6b927 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 4 Dec 2016 20:05:36 +0100 Subject: fix #283 : implement LZ4_versionString(). --- lib/lz4.c | 1 + programs/lz4cli.c | 2 +- tests/fuzzer.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index fc19493..143c36e 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -391,6 +391,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } diff --git a/programs/lz4cli.c b/programs/lz4cli.c index a452d1c..46c2f1b 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -78,7 +78,7 @@ ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR +#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR #define LZ4_EXTENSION ".lz4" #define LZ4CAT "lz4cat" #define UNLZ4 "unlz4" diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 06cfd40..260208a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1102,7 +1102,7 @@ int main(int argc, const char** argv) } } - printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); + printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString()); if (!seedset) { time_t const t = time(NULL); -- cgit v0.12 From 3f430daf7aa6ce0a96a863387de33be54d45806e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 15:20:59 +0100 Subject: added lz4opt.h --- lib/lz4opt.h | 573 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 573 insertions(+) create mode 100644 lib/lz4opt.h diff --git a/lib/lz4opt.h b/lib/lz4opt.h new file mode 100644 index 0000000..990cc67 --- /dev/null +++ b/lib/lz4opt.h @@ -0,0 +1,573 @@ +/* + lz4opt.h - Optimal Mode of LZ4 + Copyright (C) 2015-2016, Przemyslaw Skibinski + Note : this file is intended to be included within lz4hc.c + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) + + +#define LZ4_OPT_NUM (1<<12) + +typedef struct +{ + int off; + int len; + int back; +} LZ4HC_match_t; + +typedef struct +{ + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + + +FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) +{ + size_t price = 8*litlen; + if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + return price; +} + +FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t offset, size_t mlen) +{ + size_t price = 16 + 8; /* 16-bit offset + token */ + + price += 8*litlen; + if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + + // mlen-=MINMATCH; + if (mlen>=(int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + + return price; +} + + +FORCE_INLINE int LZ4HC_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ4HC_match_t* matches) +{ + U16* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + U32 matchIndex; + int nbAttempts = ctx->searchNum; + int mnum = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ4HC_hashPtr(ip)]; + matchIndex = *HashPos; + + + DELTANEXTU16(current) = (U16)(current - matchIndex); + *HashPos = current; + ctx->nextToUpdate++; + + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if ((ip[best_mlen] == match[best_mlen]) && (LZ4_read32(match) == LZ4_read32(ip))) + { + size_t mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + + while ((ip+back>iLowLimit) + && (match+back > lowPrefixPtr) + && (ip[back-1] == match[back-1])) + back--; + + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + matchIndex -= DELTANEXTU16(matchIndex); + } + + + return mnum; +} + + + +FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ4HC_match_t* matches) +{ + U16* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + int nbAttempts = ctx->searchNum; + int mnum = 0; + U16 *ptr0, *ptr1; + U32 matchIndex, delta0, delta1; + size_t mlt = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ4HC_hashPtr(ip)]; + matchIndex = *HashPos; + + *HashPos = current; + ctx->nextToUpdate++; + + // check rest of matches + ptr0 = &DELTANEXTU16(current*2+1); + ptr1 = &DELTANEXTU16(current*2); + delta0 = delta1 = current - matchIndex; + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + mlt = 0; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if (LZ4_read32(match) == LZ4_read32(ip)) + { + mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ4_OPT_NUM) break; + } + } + + if (*(ip+mlt) < *(match+mlt)) + { + *ptr0 = delta0; + ptr0 = &DELTANEXTU16(matchIndex*2); + // printf("delta0=%d\n", delta0); + if (*ptr0 == (U16)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } + else + { + *ptr1 = delta1; + ptr1 = &DELTANEXTU16(matchIndex*2+1); + // printf("delta1=%d\n", delta1); + if (*ptr1 == (U16)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U16)-1; + *ptr1 = (U16)-1; + + return mnum; +} + + + + + +#define SET_PRICE(pos, mlen, offset, litlen, price) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + +static int LZ4HC_compress_optimal ( + LZ4HC_CCtx_internal* ctx, + const BYTE* source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive limit, + const int binaryTreeFinder, + const size_t sufficient_len, + const int faster_get_matches + ) +{ + LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; + LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2, skip_num = 0; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + + /* init */ + ctx->end += inputSize; + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + memset(opt, 0, sizeof(LZ4HC_optimal_t)); + last_pos = 0; + llen = ip - anchor; + + best_mlen = (last_pos) ? last_pos : (MINMATCH-1); + + if (faster_get_matches && last_pos) + match_num = 0; + else + { + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, ip); + match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); + } + } + + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (size_t)matches[match_num-1].len > sufficient_len) + { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto encode; + } + + // set prices using matches at position = 0 + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; + LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + while (mlen <= best_mlen) + { + litlen = 0; + price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + mlen++; + } + } + + if (last_pos < MINMATCH) { ip++; continue; } + + opt[0].mlen = opt[1].mlen = 1; + + // check further positions + for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) + { + litlen = opt[cur-1].litlen + 1; + + if (cur != litlen) + { + price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); + LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } + else + { + price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; + LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } + else + { + litlen = 1; + price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); + } + + mlen = 1; + best_mlen = 0; + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, mlen, best_mlen, litlen, price); + + if (cur == last_pos) break; + + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + + + if (faster_get_matches && skip_num > 0) + { + skip_num--; + continue; + } + + + best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); + + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, inr); + match_num = LZ4HC_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + + + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) + { + cur -= matches[match_num-1].back; + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto encode; + } + + // set prices using matches at position = cur + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + cur2 = cur - matches[i].back; + best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; + LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + + if (mlen < (size_t)matches[i].back + 1) + mlen = matches[i].back + 1; + + while (mlen <= best_mlen) + { + if (opt[cur2].mlen == 1) + { + litlen = opt[cur2].litlen; + + if (cur2 != litlen) + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + else + price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + } + else + { + litlen = 0; + price = opt[cur2].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + } + + LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) + { + SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + } + + mlen++; + } + } + } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + +encode: // cur, last_pos, best_mlen, best_off have to be set + for (i = 1; i <= last_pos; i++) + { + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + } + + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + + opt[0].mlen = 1; + + while (1) + { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = (int)best_mlen; + opt[cur].off = (int)best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (i = 0; i <= last_pos;) + { + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + i += opt[i].mlen; + } + + cur = 0; + + while (cur < last_pos) + { + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + + LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); + LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); + + if (res) return 0; + + LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); + } + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + // if (inputSize > LASTLITERALS && lastRun < LASTLITERALS) { printf("ERROR: lastRun=%d\n", lastRun); } + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun< Date: Tue, 6 Dec 2016 15:21:28 +0100 Subject: introduced LZ4HC_compress_hashChain --- lib/lz4hc.c | 30 +++++++++++++++++++++++++++--- lib/lz4hc.h | 8 +++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 298550c..57587f3 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -305,8 +305,9 @@ FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } +#include "lz4opt.h" -static int LZ4HC_compress_generic ( +static int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, @@ -336,8 +337,6 @@ static int LZ4HC_compress_generic ( const BYTE* ref0; /* init */ - if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL; - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; @@ -490,6 +489,31 @@ _Search3: } +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int const inputSize, + int const maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) +{ + if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; + if (compressionLevel > 16) { + switch (compressionLevel) { + case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64, 0); + case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256, 0); + case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64, 0); + case 20: + default: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 256, 0); + } + } + + return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limit); +} + + int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index d8ac3c0..40ca401 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,7 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 16 +#define LZ4HC_MAX_CLEVEL 20 /*-************************************ @@ -58,7 +58,7 @@ extern "C" { * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. - * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL. * @return : the number of bytes written into 'dst' * or 0 if compression fails. */ @@ -153,6 +153,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -169,12 +170,13 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; #endif -#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE 262200 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From 2113ead176e0032c7ba04aa93f3bcc3d04ba6142 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 19:11:53 +0100 Subject: 17-bit LZ4HC_MAXD --- lib/lz4hc.c | 16 ++++++++++------ lib/lz4hc.h | 4 ++-- lib/lz4opt.h | 51 ++++++++++++++++++--------------------------------- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 57587f3..abdef68 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,7 +84,7 @@ * Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -499,14 +499,18 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { +/* +16#silesia_tar : 211947520 -> 77841782 (2.723), 9.8 MB/s ,1874.4 MB/s + +*/ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 16) { switch (compressionLevel) { - case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64, 0); - case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256, 0); - case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64, 0); - case 20: - default: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 256, 0); + case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64); + case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256); + case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64); + default: + case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, LZ4_OPT_NUM); } } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 40ca401..4882d66 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -131,7 +131,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ #define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<nextToUpdate++; // check rest of matches - ptr0 = &DELTANEXTU16(current*2+1); - ptr1 = &DELTANEXTU16(current*2); + ptr0 = &DELTANEXTMAXD(current*2+1); + ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = current - matchIndex; while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) @@ -265,7 +265,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; - ptr0 = &DELTANEXTU16(matchIndex*2); + ptr0 = &DELTANEXTMAXD(matchIndex*2); // printf("delta0=%d\n", delta0); if (*ptr0 == (U16)-1) break; delta0 = *ptr0; @@ -275,7 +275,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( else { *ptr1 = delta1; - ptr1 = &DELTANEXTU16(matchIndex*2+1); + ptr1 = &DELTANEXTMAXD(matchIndex*2+1); // printf("delta1=%d\n", delta1); if (*ptr1 == (U16)-1) break; delta1 = *ptr1; @@ -313,14 +313,13 @@ static int LZ4HC_compress_optimal ( int maxOutputSize, limitedOutput_directive limit, const int binaryTreeFinder, - const size_t sufficient_len, - const int faster_get_matches + const size_t sufficient_len ) { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; const BYTE *inr; - size_t res, cur, cur2, skip_num = 0; + size_t res, cur, cur2; size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; const BYTE* ip = (const BYTE*) source; @@ -330,7 +329,6 @@ static int LZ4HC_compress_optimal ( const BYTE* const matchlimit = (iend - LASTLITERALS); BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - /* init */ ctx->end += inputSize; @@ -343,22 +341,17 @@ static int LZ4HC_compress_optimal ( last_pos = 0; llen = ip - anchor; - best_mlen = (last_pos) ? last_pos : (MINMATCH-1); + best_mlen = MINMATCH-1; - if (faster_get_matches && last_pos) - match_num = 0; - else - { - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, ip); - match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - } - } + if (!binaryTreeFinder) + { + LZ4HC_Insert(ctx, ip); + match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); + } + else + { + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); + } LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -393,7 +386,7 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; // check further positions - for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; @@ -430,14 +423,6 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - - if (faster_get_matches && skip_num > 0) - { - skip_num--; - continue; - } - - best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); if (!binaryTreeFinder) @@ -500,7 +485,7 @@ static int LZ4HC_compress_optimal ( mlen++; } } - } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + } // for (cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; -- cgit v0.12 From cfe6a24b0844dffa79f4a7749e076aadb4497f75 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 6 Dec 2016 19:42:47 +0100 Subject: updated LZ4HC_get_price --- lib/lz4opt.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index fbd3048..5208438 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -63,7 +63,7 @@ FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) return price; } -FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t offset, size_t mlen) +FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ @@ -307,7 +307,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, - const BYTE* source, + const char* const source, char* dest, int inputSize, int maxOutputSize, @@ -374,7 +374,7 @@ static int LZ4HC_compress_optimal ( while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; @@ -465,14 +465,14 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); else - price = LZ4HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); } LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); -- cgit v0.12 From 757ef1d5fc710598c8a7325f9b227ccd96692187 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 11:44:17 +0100 Subject: removed LZ4HC_GetAllMatches --- lib/lz4hc.c | 8 ++-- lib/lz4opt.h | 142 +++++------------------------------------------------------ 2 files changed, 15 insertions(+), 135 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index abdef68..4381238 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -506,11 +506,11 @@ static int LZ4HC_compress_generic ( if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 16) { switch (compressionLevel) { - case 17: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 64); - case 18: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 0, 256); - case 19: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, 64); + case 17: ctx->searchNum = 16; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 16); + case 18: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 64); + case 19: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); default: - case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 1, LZ4_OPT_NUM); + case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); } } diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 5208438..0bffa3b 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -42,17 +42,17 @@ typedef struct { - int off; - int len; - int back; + int off; + int len; + int back; } LZ4HC_match_t; typedef struct { - int price; - int off; - int mlen; - int litlen; + int price; + int off; + int mlen; + int litlen; } LZ4HC_optimal_t; @@ -63,6 +63,7 @@ FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) return price; } + FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ @@ -77,107 +78,6 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) } -FORCE_INLINE int LZ4HC_GetAllMatches ( - LZ4HC_CCtx_internal* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - size_t best_mlen, - LZ4HC_match_t* matches) -{ - U16* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); - const BYTE* const dictBase = ctx->dictBase; - const BYTE* match; - U32 matchIndex; - int nbAttempts = ctx->searchNum; - int mnum = 0; - U32* HashPos; - - if (ip + MINMATCH > iHighLimit) return 0; - - /* First Match */ - HashPos = &HashTable[LZ4HC_hashPtr(ip)]; - matchIndex = *HashPos; - - - DELTANEXTU16(current) = (U16)(current - matchIndex); - *HashPos = current; - ctx->nextToUpdate++; - - - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) - { - nbAttempts--; - if (matchIndex >= dictLimit) - { - match = base + matchIndex; - - if ((ip[best_mlen] == match[best_mlen]) && (LZ4_read32(match) == LZ4_read32(ip))) - { - size_t mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - int back = 0; - - while ((ip+back>iLowLimit) - && (match+back > lowPrefixPtr) - && (ip[back-1] == match[back-1])) - back--; - - mlt -= back; - - if (mlt > best_mlen) - { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - } - - if (best_mlen > LZ4_OPT_NUM) break; - } - } - else - { - match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { - size_t mlt; - int back=0; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if (mlt > best_mlen) - { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - } - - if (best_mlen > LZ4_OPT_NUM) break; - } - } - matchIndex -= DELTANEXTU16(matchIndex); - } - - - return mnum; -} - - - FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, @@ -312,7 +212,6 @@ static int LZ4HC_compress_optimal ( int inputSize, int maxOutputSize, limitedOutput_directive limit, - const int binaryTreeFinder, const size_t sufficient_len ) { @@ -342,16 +241,7 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; best_mlen = MINMATCH-1; - - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, ip); - match_num = LZ4HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - } + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } @@ -425,18 +315,8 @@ static int LZ4HC_compress_optimal ( best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); - if (!binaryTreeFinder) - { - LZ4HC_Insert(ctx, inr); - match_num = LZ4HC_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); - LZ4_LOG_PARSER("%d: LZ4HC_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } - else - { - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } - + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { -- cgit v0.12 From e3fee94742f9027aa2d27fae6458ce285889552e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:16:33 +0100 Subject: LZ4HC_MAX_CLEVEL = 12 --- lib/lz4hc.c | 16 ++++++---------- lib/lz4hc.h | 4 ++-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 4381238..608dea9 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -313,7 +313,7 @@ static int LZ4HC_compress_hashChain ( char* const dest, int const inputSize, int const maxOutputSize, - int compressionLevel, + unsigned maxNbAttempts, limitedOutput_directive limit ) { @@ -326,7 +326,6 @@ static int LZ4HC_compress_hashChain ( BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; - unsigned maxNbAttempts; int ml, ml2, ml3, ml0; const BYTE* ref = NULL; const BYTE* start2 = NULL; @@ -337,7 +336,6 @@ static int LZ4HC_compress_hashChain ( const BYTE* ref0; /* init */ - maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; ip++; @@ -504,17 +502,15 @@ static int LZ4HC_compress_generic ( */ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; - if (compressionLevel > 16) { + if (compressionLevel > 9) { switch (compressionLevel) { - case 17: ctx->searchNum = 16; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 16); - case 18: ctx->searchNum = 64; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 64); - case 19: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); + case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); + case 11: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); default: - case 20: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); + case 12: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); } } - - return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limit); + return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 4882d66..fc8067b 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,7 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MAX_CLEVEL 20 +#define LZ4HC_MAX_CLEVEL 12 /*-************************************ @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE+2) #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) -- cgit v0.12 From 77b051ed7bf383b97db43e4fc4b523df3b003e8a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:17:24 +0100 Subject: improve code formatting --- lib/lz4hc.c | 6 ++-- lib/lz4opt.h | 100 ++++++++++++++++++++--------------------------------------- 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 608dea9..9dc6be1 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -498,8 +498,10 @@ static int LZ4HC_compress_generic ( ) { /* -16#silesia_tar : 211947520 -> 77841782 (2.723), 9.8 MB/s ,1874.4 MB/s - + 9#silesia_tar : 211947520 -> 77891907 (2.721), 24.3 MB/s ,2142.1 MB/s +10#silesia_tar : 211947520 -> 77841782 (2.723), 11.4 MB/s ,2185.3 MB/s +11#silesia_tar : 211947520 -> 77408334 (2.738), 6.1 MB/s ,2288.9 MB/s +12#silesia_tar : 211947520 -> 77319973 (2.741), 3.3 MB/s ,2361.0 MB/s */ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 9) { diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 0bffa3b..2eaba4f 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -105,7 +105,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( /* First Match */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; - *HashPos = current; ctx->nextToUpdate++; @@ -114,20 +113,15 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = current - matchIndex; - while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) - { + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; mlt = 0; if (matchIndex >= dictLimit) { match = base + matchIndex; - - if (LZ4_read32(match) == LZ4_read32(ip)) - { + if (LZ4_read32(match) == LZ4_read32(ip)) { mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - - if (mlt > best_mlen) - { + if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); matches[mnum].len = (int)mlt; @@ -141,16 +135,14 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( else { match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { + if (LZ4_read32(match) == LZ4_read32(ip)) { const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - if (mlt > best_mlen) - { + if (mlt > best_mlen) { best_mlen = mlt; matches[mnum].off = (int)(ip - match); matches[mnum].len = (int)mlt; @@ -162,21 +154,18 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } } - if (*(ip+mlt) < *(match+mlt)) - { + if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - // printf("delta0=%d\n", delta0); + /* printf("delta0=%d\n", delta0); */ if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; - } - else - { + } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - // printf("delta1=%d\n", delta1); + /* printf("delta1=%d\n", delta1); */ if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; @@ -234,8 +223,7 @@ static int LZ4HC_compress_optimal ( ip++; /* Main Loop */ - while (ip < mflimit) - { + while (ip < mflimit) { memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; @@ -256,13 +244,11 @@ static int LZ4HC_compress_optimal ( } // set prices using matches at position = 0 - for (i = 0; i < match_num; i++) - { + for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); - while (mlen <= best_mlen) - { + while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) @@ -276,40 +262,33 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; // check further positions - for (cur = 1; cur <= last_pos; cur++) - { - inr = ip + cur; + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; - if (opt[cur-1].mlen == 1) - { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur != litlen) - { + if (cur != litlen) { price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } - else - { + } else { price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } - } - else - { + } else { litlen = 1; price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); - } + } - mlen = 1; - best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + mlen = 1; + best_mlen = 0; + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); - if (cur == last_pos) break; + if (cur == last_pos) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); @@ -318,8 +297,7 @@ static int LZ4HC_compress_optimal ( match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) - { + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -328,8 +306,7 @@ static int LZ4HC_compress_optimal ( } // set prices using matches at position = cur - for (i = 0; i < match_num; i++) - { + for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; cur2 = cur - matches[i].back; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; @@ -338,27 +315,22 @@ static int LZ4HC_compress_optimal ( if (mlen < (size_t)matches[i].back + 1) mlen = matches[i].back + 1; - while (mlen <= best_mlen) - { - if (opt[cur2].mlen == 1) - { + while (mlen <= best_mlen) { + if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 != litlen) price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); else price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; - } - else - { + } else { litlen = 0; price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); } LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) - if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) - { + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } @@ -373,8 +345,7 @@ static int LZ4HC_compress_optimal ( cur = last_pos - best_mlen; encode: // cur, last_pos, best_mlen, best_off have to be set - for (i = 1; i <= last_pos; i++) - { + for (i = 1; i <= last_pos; i++) { LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } @@ -382,8 +353,7 @@ encode: // cur, last_pos, best_mlen, best_off have to be set opt[0].mlen = 1; - while (1) - { + while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; opt[cur].mlen = (int)best_mlen; @@ -394,16 +364,14 @@ encode: // cur, last_pos, best_mlen, best_off have to be set cur -= mlen; } - for (i = 0; i <= last_pos;) - { + for (i = 0; i <= last_pos;) { LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; - while (cur < last_pos) - { + while (cur < last_pos) { LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } -- cgit v0.12 From f2ebf37bfe64e24a6f8833dfdd206266cae25f6f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:49:38 +0100 Subject: slightly improved ratio --- lib/lz4opt.h | 110 ++++++++++++++++++++++------------------------------------- 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 2eaba4f..cc51ec5 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -44,7 +44,6 @@ typedef struct { int off; int len; - int back; } LZ4HC_match_t; typedef struct @@ -56,7 +55,7 @@ typedef struct } LZ4HC_optimal_t; -FORCE_INLINE size_t LZ4_LIT_ONLY_COST(size_t litlen) +FORCE_INLINE size_t LZ4HC_GetLiteralsPrice(size_t litlen) { size_t price = 8*litlen; if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } @@ -69,10 +68,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) size_t price = 16 + 8; /* 16-bit offset + token */ price += 8*litlen; - if (litlen>=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } - // mlen-=MINMATCH; - if (mlen>=(int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + mlen -= MINMATCH; + if (mlen >= (int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } return price; } @@ -119,37 +118,28 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (matchIndex >= dictLimit) { match = base + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - mlt = MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - } - + mlt = LZ4_count(ip, match, iHighLimit); + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; if (best_mlen > LZ4_OPT_NUM) break; } } else { match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) { - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - } - + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip, match, vLimit); + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; if (best_mlen > LZ4_OPT_NUM) break; } } @@ -227,30 +217,26 @@ static int LZ4HC_compress_optimal ( memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches); + if (!match_num) { ip++; continue; } + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); - best_mlen = MINMATCH-1; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); - - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } - - if (match_num && (size_t)matches[match_num-1].len > sufficient_len) - { + if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; last_pos = 1; goto encode; - } + } - // set prices using matches at position = 0 - for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + /* set prices using matches at position = 0 */ + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; if (mlen > last_pos || price < (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; @@ -261,44 +247,38 @@ static int LZ4HC_compress_optimal ( opt[0].mlen = opt[1].mlen = 1; - // check further positions + /* check further positions */ for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur != litlen) { - price = opt[cur - litlen].price + LZ4_LIT_ONLY_COST(litlen); + price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4_LIT_ONLY_COST(llen + litlen) - llen; + price = LZ4HC_GetLiteralsPrice(llen + litlen) - 8*llen; LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LZ4_LIT_ONLY_COST(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4_LIT_ONLY_COST(litlen)); + price = opt[cur - 1].price + LZ4HC_GetLiteralsPrice(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_GetLiteralsPrice(litlen)); } mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - best_mlen = (best_mlen > MINMATCH) ? best_mlen : (MINMATCH-1); - - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { - cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; last_pos = cur + 1; @@ -307,39 +287,33 @@ static int LZ4HC_compress_optimal ( // set prices using matches at position = cur for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; - cur2 = cur - matches[i].back; + mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; + cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); - if (mlen < (size_t)matches[i].back + 1) - mlen = matches[i].back + 1; - while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen - MINMATCH); + price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen - MINMATCH) - llen; + price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, mlen - MINMATCH); + price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); - // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } - mlen++; } } } // for (cur = 1; cur <= last_pos; cur++) - best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; @@ -352,7 +326,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); opt[0].mlen = 1; - while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; @@ -370,7 +343,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set } cur = 0; - while (cur < last_pos) { LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; -- cgit v0.12 From 1833be1cf0e545f8b24bb4a786e274a9e419e280 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 12:59:05 +0100 Subject: fixed gcc warnings --- lib/lz4hc.c | 2 +- lib/lz4opt.h | 16 +++++++--------- programs/lz4io.c | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 9dc6be1..de7a55c 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,7 +84,7 @@ * Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) -#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ +#define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } diff --git a/lib/lz4opt.h b/lib/lz4opt.h index cc51ec5..6922e35 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -94,8 +94,8 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( const BYTE* match; int nbAttempts = ctx->searchNum; int mnum = 0; - U16 *ptr0, *ptr1; - U32 matchIndex, delta0, delta1; + U16 *ptr0, *ptr1, delta0, delta1; + U32 matchIndex; size_t mlt = 0; U32* HashPos; @@ -129,8 +129,8 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } else { - match = dictBase + matchIndex; const BYTE* vLimit = ip + (dictLimit - matchIndex); + match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip, match, vLimit); if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) @@ -245,9 +245,8 @@ static int LZ4HC_compress_optimal ( if (last_pos < MINMATCH) { ip++; continue; } - opt[0].mlen = opt[1].mlen = 1; - /* check further positions */ + opt[0].mlen = opt[1].mlen = 1; for (cur = 1; cur <= last_pos; cur++) { inr = ip + cur; @@ -285,7 +284,7 @@ static int LZ4HC_compress_optimal ( goto encode; } - // set prices using matches at position = cur + /* set prices using matches at position = cur */ for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; @@ -312,13 +311,13 @@ static int LZ4HC_compress_optimal ( mlen++; } } - } // for (cur = 1; cur <= last_pos; cur++) + } /* for (cur = 1; cur <= last_pos; cur++) */ best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; -encode: // cur, last_pos, best_mlen, best_off have to be set +encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } @@ -363,7 +362,6 @@ encode: // cur, last_pos, best_mlen, best_off have to be set /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - // if (inputSize > LASTLITERALS && lastRun < LASTLITERALS) { printf("ERROR: lastRun=%d\n", lastRun); } if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Wed, 7 Dec 2016 13:07:39 +0100 Subject: improved LZ4HC_BinTree_GetAllMatches --- lib/lz4opt.h | 66 ++++++++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 6922e35..a2a9d0b 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -110,40 +110,30 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( // check rest of matches ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); - delta0 = delta1 = current - matchIndex; + delta0 = delta1 = (U16)(current - matchIndex); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; - mlt = 0; - if (matchIndex >= dictLimit) - { + if (matchIndex >= dictLimit) { match = base + matchIndex; mlt = LZ4_count(ip, match, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - mnum++; - if (best_mlen > LZ4_OPT_NUM) break; - } - } - else - { + } else { const BYTE* vLimit = ip + (dictLimit - matchIndex); match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip, match, vLimit); if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - mnum++; - if (best_mlen > LZ4_OPT_NUM) break; - } } - + + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + mnum++; + if (best_mlen > LZ4_OPT_NUM) break; + } + if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); @@ -170,18 +160,15 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } - - - -#define SET_PRICE(pos, mlen, offset, litlen, price) \ - { \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = (int)mlen; \ - opt[pos].off = (int)offset; \ - opt[pos].litlen = (int)litlen; \ - opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ - } +#define SET_PRICE(pos, mlen, offset, litlen, price) \ +{ \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ +} static int LZ4HC_compress_optimal ( @@ -194,11 +181,11 @@ static int LZ4HC_compress_optimal ( const size_t sufficient_len ) { - LZ4HC_optimal_t opt[LZ4_OPT_NUM + 4]; - LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; - const BYTE *inr; - size_t res, cur, cur2; - size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; + LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; @@ -373,4 +360,3 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ /* End */ return (int) ((char*)op-dest); } - -- cgit v0.12 From 1289038240ab81aee4cc70ef071da907ccfdfcde Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 14:31:46 +0100 Subject: 15-bit LZ4HC_HASH_LOG --- lib/lz4hc.h | 2 +- programs/lz4io.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index fc8067b..0523314 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) -#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE+2) +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) diff --git a/programs/lz4io.c b/programs/lz4io.c index f8a8f2b..46daaf7 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -591,9 +591,10 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t ress; if (dstFileName == NULL) return ifntSize; /* not enough memory */ + ress = LZ4IO_createCResources(); /* loop on each file */ for (i=0; i Date: Wed, 7 Dec 2016 15:49:45 +0100 Subject: improved ratio --- lib/lz4hc.h | 2 +- lib/lz4opt.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 0523314..5eafcd6 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -131,7 +131,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ #define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE+1)) +#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE)) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) #define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index a2a9d0b..1ff3375 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -224,7 +224,7 @@ static int LZ4HC_compress_optimal ( while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; - if (mlen > last_pos || price < (size_t)opt[mlen].price) + if (mlen > last_pos || price <= (size_t)opt[mlen].price) SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } @@ -255,7 +255,7 @@ static int LZ4HC_compress_optimal ( mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price < (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; -- cgit v0.12 From 76228f064a95f7135e5990387b23002dd296b2da Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 16:19:10 +0100 Subject: LZ4HC_DICTIONARY_LOGSIZE 17 --- lib/lz4hc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 5eafcd6..4ff1d17 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -130,11 +130,11 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in * They are exposed to allow static allocation of `LZ4_streamHC_t`. * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 **************************************/ -#define LZ4HC_DICTIONARY_LOGSIZE 16 -#define LZ4HC_MAXD (1<<(LZ4HC_DICTIONARY_LOGSIZE)) +#define LZ4HC_DICTIONARY_LOGSIZE 17 +#define LZ4HC_MAXD (1< Date: Wed, 7 Dec 2016 16:24:35 +0100 Subject: fixed LZ4_STREAMHCSIZE --- lib/lz4hc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 4ff1d17..35e88b6 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -176,7 +176,7 @@ typedef struct #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 52) /* 393268 */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From 01ffed25adbb4d18e48ceac59b1e5174ae29fc7f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 14:29:22 -0800 Subject: fix : invocation of path/to/unlz4 (#284), reported by @beiDei8z --- programs/Makefile | 21 ++++++++++++--------- programs/lz4cli.c | 9 ++++++++- tests/Makefile | 15 +++++++++++++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index ffbcbf1..84662c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,7 +91,7 @@ lz4c32: lz4 clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 @echo Cleaning completed @@ -112,13 +112,16 @@ preview-man: clean-man man #FreeBSD targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +unlz4: lz4 + ln -s lz4 unlz4 + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 + @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @@ -127,10 +130,10 @@ install: lz4$(EXT) lz4c$(EXT) @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/lz4cat + @$(RM) $(DESTDIR)$(BINDIR)/unlz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4c @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 46c2f1b..eda2018 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -260,6 +260,13 @@ static void waitEnter(void) (void)getchar(); } +static const char* lastNameFromPath(const char* path) +{ + const char* name = strrchr(path, '/'); + if (name==NULL) name = strrchr(path, '\\'); /* windows */ + if (name==NULL) name = path-1; + return name+1; +} /*! readU32FromChar() : @return : unsigned integer value reach from input in `char` format @@ -294,7 +301,7 @@ int main(int argc, const char** argv) const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = argv[0]; + const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; diff --git a/tests/Makefile b/tests/Makefile index e58d449..5923a20 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -206,12 +206,15 @@ test-lz4-multiple: lz4 datagen $(LZ4) -f -m tmp1 notHere tmp2; echo $$? @$(RM) tmp* -test-lz4-basic: lz4 datagen +unlz4: + @$(MAKE) -C $(PRGDIR) unlz4 + +test-lz4-basic: lz4 datagen unlz4 @echo "\n ---- test lz4 basic compression/decompression ----" ./datagen -g0 | $(LZ4) -v | $(LZ4) -t ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t ./datagen -g20KB > tmpSrc - $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes + $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes diff -q tmpSrc tmpRes $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes diff -q tmpSrc tmpRes @@ -220,6 +223,14 @@ test-lz4-basic: lz4 datagen ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + @echo "hello world" > tmp + $(LZ4) --rm tmp + ls -ls tmp && false || true # must fail (--rm) + ls -ls tmp.lz4 + $(PRGDIR)/unlz4 --rm tmp.lz4 + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (--rm) + ls -ls tmp.lz4.lz4 && false || true # must fail (unlz4) @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 750a50cf67e8574500e4f9b4bb5e7f6cdd6f9c26 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 14:49:04 -0800 Subject: fix #284, reported by @beiDei8z : added test cases for path/to/lz4cat --- programs/.gitignore | 2 ++ programs/Makefile | 5 ++++- programs/lz4cli.c | 1 + tests/Makefile | 14 +++++++++++--- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/programs/.gitignore b/programs/.gitignore index df3423e..daa7f14 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,5 +1,7 @@ # local binary (Makefile) lz4 +unlz4 +lz4cat lz4c lz4c32 datagen diff --git a/programs/Makefile b/programs/Makefile index 84662c2..74a2440 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,7 +91,7 @@ lz4c32: lz4 clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat @echo Cleaning completed @@ -115,6 +115,9 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) unlz4: lz4 ln -s lz4 unlz4 +lz4cat: lz4 + ln -s lz4 lz4cat + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index eda2018..76e7e14 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -319,6 +319,7 @@ int main(int argc, const char** argv) if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); + LZ4IO_setRemoveSrcFile(0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; diff --git a/tests/Makefile b/tests/Makefile index 5923a20..adfcfea 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -207,9 +207,12 @@ test-lz4-multiple: lz4 datagen @$(RM) tmp* unlz4: - @$(MAKE) -C $(PRGDIR) unlz4 + @$(MAKE) -C $(PRGDIR) $@ -test-lz4-basic: lz4 datagen unlz4 +lz4cat: + @$(MAKE) -C $(PRGDIR) $@ + +test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" ./datagen -g0 | $(LZ4) -v | $(LZ4) -t ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t @@ -224,13 +227,18 @@ test-lz4-basic: lz4 datagen unlz4 ./datagen -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t ./datagen -g256MB | $(LZ4) -vqB4D | $(LZ4) -t @echo "hello world" > tmp - $(LZ4) --rm tmp + $(LZ4) --rm -f tmp ls -ls tmp && false || true # must fail (--rm) ls -ls tmp.lz4 + $(PRGDIR)/lz4cat tmp.lz4 # must display hello world + ls -ls tmp.lz4 $(PRGDIR)/unlz4 --rm tmp.lz4 ls -ls tmp ls -ls tmp.lz4 && false || true # must fail (--rm) ls -ls tmp.lz4.lz4 && false || true # must fail (unlz4) + $(PRGDIR)/lz4cat tmp # pass-through mode + ls -ls tmp + ls -ls tmp.lz4 && false || true # must fail (lz4cat) @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 7e2d4c4c08934a1fa85fa8324c3e134ce69c6648 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 7 Dec 2016 15:52:07 -0800 Subject: fixed test-lz4c32 --- tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index adfcfea..eb36abf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -38,7 +38,7 @@ VOID := /dev/null TESTDIR := versionsTest PYTHON ?= python3 -CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make +CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 @@ -207,10 +207,10 @@ test-lz4-multiple: lz4 datagen @$(RM) tmp* unlz4: - @$(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" lz4cat: - @$(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)" test-lz4-basic: lz4 datagen unlz4 lz4cat @echo "\n ---- test lz4 basic compression/decompression ----" -- cgit v0.12 From 0280cf40dd01caa784d291480aaead9987ae9f9a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 02:47:42 +0100 Subject: fixed path-1, as suggested by @t-mat --- programs/lz4cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 76e7e14..b63d69d 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -264,7 +264,7 @@ static const char* lastNameFromPath(const char* path) { const char* name = strrchr(path, '/'); if (name==NULL) name = strrchr(path, '\\'); /* windows */ - if (name==NULL) name = path-1; + if (name==NULL) return path; return name+1; } -- cgit v0.12 From fb6c98c856d300b39baa40f8408ab639e6e9b56a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 8 Dec 2016 15:50:45 +0100 Subject: slightly improved lz4opt.h --- lib/lz4hc.h | 2 +- lib/lz4opt.h | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 35e88b6..59d912e 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -134,7 +134,7 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_MAXD (1<=(int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); return price; } @@ -68,10 +69,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) size_t price = 16 + 8; /* 16-bit offset + token */ price += 8*litlen; - if (litlen >= (int)RUN_MASK) { litlen-=RUN_MASK; price+=8*(1+litlen/255); } + if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); mlen -= MINMATCH; - if (mlen >= (int)ML_MASK) { mlen-=ML_MASK; price+=8*(1+mlen/255); } + if (mlen >= (int)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); return price; } @@ -107,7 +108,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( *HashPos = current; ctx->nextToUpdate++; - // check rest of matches ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); @@ -137,7 +137,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (*(ip+mlt) < *(match+mlt)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - /* printf("delta0=%d\n", delta0); */ if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; @@ -145,7 +144,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - /* printf("delta1=%d\n", delta1); */ if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; @@ -155,7 +153,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; - return mnum; } @@ -223,9 +220,8 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; - if (mlen > last_pos || price <= (size_t)opt[mlen].price) - SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } } @@ -243,7 +239,7 @@ static int LZ4HC_compress_optimal ( price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4HC_GetLiteralsPrice(llen + litlen) - 8*llen; + price = LZ4HC_GetLiteralsPrice(llen + litlen) - LZ4HC_GetLiteralsPrice(llen); LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { @@ -255,7 +251,7 @@ static int LZ4HC_compress_optimal ( mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - if (cur > last_pos || price < (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos) break; @@ -285,14 +281,14 @@ static int LZ4HC_compress_optimal ( if (cur2 != litlen) price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen) - 8*llen; + price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); } else { litlen = 0; price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); } LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); - if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } mlen++; -- cgit v0.12 From de93e9e5d84fef6c5741ff0d72306fdb2baade1e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 14:17:40 -0800 Subject: fix #285 : lz4cat doesn't work with stdin (reported by @beiDei8z) --- programs/lz4.1.md | 8 +++++--- programs/lz4cli.c | 13 ++++++++----- tests/Makefile | 5 +++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 83a08b6..48f3152 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -34,6 +34,7 @@ Differences are : * `lz4` preserves original files * `lz4` compresses a single file by default (see `-m` for multiple files) * `lz4 file1 file2` means : compress file1 _into_ file2 + * `lz4 file.lz4` will default to decompression (use `-z` to force compression) * `lz4` shows real-time notification statistics during compression or decompression of a single file (use `-q` to silent them) @@ -43,15 +44,16 @@ Differences are : `file` is compressed into `file.lz4`. * As a consequence of previous rules, note the following example : `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, - hence it does _not_ create any `file.lz4`. + hence it does _not_ create `file.lz4`. Default behaviors can be modified by opt-in commands, detailed below. * `lz4 -m` makes it possible to provide multiple input filenames, which will be compressed into files using suffix `.lz4`. - Progress notifications are also disabled by default. + Progress notifications are also disabled by default (use `-v` to enable them). This mode has a behavior which more closely mimics `gzip` command line, - with the main difference being that source files are preserved by default. + with the main remaining difference being that source files are preserved by default. + * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. * It's possible to opt-in to erase source files on successful compression or decompression, using `--rm` command. * Consequently, `lz4 -m --rm` behaves the same as `gzip`. diff --git a/programs/lz4cli.c b/programs/lz4cli.c index b63d69d..5bd06d9 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -313,6 +313,7 @@ int main(int argc, const char** argv) DISPLAY("Allocation error : not enough memory \n"); return 1; } + inFileNames[0] = stdinmark; LZ4IO_setOverwrite(0); /* lz4cat predefined behavior */ @@ -468,7 +469,7 @@ int main(int argc, const char** argv) break; #ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ + /* recursive */ case 'r': recursive=1; /* without break */ #endif /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ @@ -482,7 +483,7 @@ int main(int argc, const char** argv) iters = readU32FromChar(&argument); argument--; BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ } break; @@ -555,6 +556,9 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + /* if input==stdin and no output defined, stdout becomes default output */ + if (!strcmp(input_filename, stdinmark) && !output_filename) + output_filename = stdoutmark; /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { @@ -592,20 +596,19 @@ int main(int argc, const char** argv) break; } - if (!output_filename) output_filename = "*\\dummy^!//"; - /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); exit(1); } - /* Downgrade notification level in stdout and multiple file mode */ if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); diff --git a/tests/Makefile b/tests/Makefile index eb36abf..01b45bc 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -239,6 +239,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat $(PRGDIR)/lz4cat tmp # pass-through mode ls -ls tmp ls -ls tmp.lz4 && false || true # must fail (lz4cat) + $(LZ4) tmp # creates tmp.lz4 + $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) + diff -q tmp tmp3 + $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) + diff -q tmp tmp2 @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 626552671a69a13cd467e27ed149dac4b35fd484 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 8 Dec 2016 14:37:12 -0800 Subject: updated NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index b879469..292cd27 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v1.7.5 cli : fix minor notification when using -r recursive mode -doc : markdown version of man page, by Takayuki Matsuoka +lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +doc : markdown version of man page, by Takayuki Matsuoka (#279) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From a22e71d4a9f4bfd0fe12acc00a7fa286faad2da6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 12:37:17 +0100 Subject: full binary tree update --- lib/lz4hc.c | 6 ------ lib/lz4opt.h | 56 ++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index de7a55c..d3578b9 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -497,12 +497,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { -/* - 9#silesia_tar : 211947520 -> 77891907 (2.721), 24.3 MB/s ,2142.1 MB/s -10#silesia_tar : 211947520 -> 77841782 (2.723), 11.4 MB/s ,2185.3 MB/s -11#silesia_tar : 211947520 -> 77408334 (2.738), 6.1 MB/s ,2288.9 MB/s -12#silesia_tar : 211947520 -> 77319973 (2.741), 3.3 MB/s ,2361.0 MB/s -*/ if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; if (compressionLevel > 9) { switch (compressionLevel) { diff --git a/lib/lz4opt.h b/lib/lz4opt.h index dc778c9..7e8150f 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -37,7 +37,6 @@ #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) - #define LZ4_OPT_NUM (1<<12) @@ -78,7 +77,10 @@ FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) } -FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( +/*-************************************* +* Binary Tree search +***************************************/ +FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, @@ -97,7 +99,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( int mnum = 0; U16 *ptr0, *ptr1, delta0, delta1; U32 matchIndex; - size_t mlt = 0; + size_t matchLength = 0; U32* HashPos; if (ip + MINMATCH > iHighLimit) return 0; @@ -106,7 +108,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; *HashPos = current; - ctx->nextToUpdate++; ptr0 = &DELTANEXTMAXD(current*2+1); ptr1 = &DELTANEXTMAXD(current*2); @@ -116,25 +117,28 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( nbAttempts--; if (matchIndex >= dictLimit) { match = base + matchIndex; - mlt = LZ4_count(ip, match, iHighLimit); + matchLength = LZ4_count(ip, match, iHighLimit); } else { const BYTE* vLimit = ip + (dictLimit - matchIndex); match = dictBase + matchIndex; if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip, match, vLimit); - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + matchLength = LZ4_count(ip, match, vLimit); + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); } - if (mlt > best_mlen) { - best_mlen = mlt; + if (matches && matchLength > best_mlen) { + best_mlen = matchLength; matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; + matches[mnum].len = (int)matchLength; mnum++; if (best_mlen > LZ4_OPT_NUM) break; } - if (*(ip+mlt) < *(match+mlt)) { + if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + + if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); if (*ptr0 == (U16)-1) break; @@ -157,6 +161,34 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( } +FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen) +{ + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + + while(idx < target) { + LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, best_mlen, NULL); + idx++; + } +} + + +/** Tree updater, providing best match */ +FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( + LZ4HC_CCtx_internal* ctx, + const BYTE* const ip, const BYTE* const iHighLimit, + size_t best_mlen, LZ4HC_match_t* matches) +{ + int mnum; + if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ + LZ4HC_updateBinTree(ctx, ip, iHighLimit, best_mlen); + mnum = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches); + ctx->nextToUpdate = (U32)(ip - ctx->base)+1; + return mnum; +} + + #define SET_PRICE(pos, mlen, offset, litlen, price) \ { \ while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ -- cgit v0.12 From d694bf91111e6227878ceda1131255f8eb5299c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 15:20:32 +0100 Subject: faster opt compression --- lib/lz4opt.h | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 7e8150f..dbc7556 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -85,7 +85,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen, - LZ4HC_match_t* matches) + LZ4HC_match_t* matches, + int* matchNum) { U16* const chainTable = ctx->chainTable; U32* const HashTable = ctx->hashTable; @@ -102,7 +103,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; - if (ip + MINMATCH > iHighLimit) return 0; + if (ip + MINMATCH > iHighLimit) return 1; /* First Match */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; @@ -127,11 +128,13 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); } - if (matches && matchLength > best_mlen) { + if (matchLength > best_mlen) { best_mlen = matchLength; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)matchLength; - mnum++; + if (matches) { + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)matchLength; + mnum++; + } if (best_mlen > LZ4_OPT_NUM) break; } @@ -157,20 +160,21 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; - return mnum; + if (matchNum) *matchNum = mnum; + // if (best_mlen > 8) return best_mlen-8; + if (!matchNum) return 8; + return 1; } -FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen) +FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdate; - while(idx < target) { - LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, best_mlen, NULL); - idx++; - } + while(idx < target) + idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -180,11 +184,11 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( const BYTE* const ip, const BYTE* const iHighLimit, size_t best_mlen, LZ4HC_match_t* matches) { - int mnum; + int mnum = 0; if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - LZ4HC_updateBinTree(ctx, ip, iHighLimit, best_mlen); - mnum = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches); - ctx->nextToUpdate = (U32)(ip - ctx->base)+1; + LZ4HC_updateBinTree(ctx, ip, iHighLimit); + best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); + ctx->nextToUpdate = (U32)(ip - ctx->base) + best_mlen; return mnum; } -- cgit v0.12 From b3ecc3e3ec768e89eea29248bd8ab0a23ddab2dd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 16:09:38 +0100 Subject: last match starts at least 12 bytes before end of block --- lib/lz4opt.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index dbc7556..c466677 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -36,6 +36,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#include #define LZ4_OPT_NUM (1<<12) @@ -161,8 +162,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; - // if (best_mlen > 8) return best_mlen-8; - if (!matchNum) return 8; + // if (best_mlen > 8) return best_mlen-8; + if (!matchNum) return 1; return 1; } @@ -290,7 +291,7 @@ static int LZ4HC_compress_optimal ( if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); - if (cur == last_pos) break; + if (cur == last_pos || inr >= mflimit) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); @@ -368,7 +369,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ offset = opt[cur].off; cur += mlen; - LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen); res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); @@ -379,12 +380,11 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ } /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); + { int lastRun = (int)(iend - anchor); if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Fri, 9 Dec 2016 17:16:35 +0100 Subject: fullUpdate as a parameter --- lib/lz4hc.c | 11 +++++++---- lib/lz4hc.h | 2 ++ lib/lz4opt.h | 23 ++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index d3578b9..e1eadf7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; + hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; @@ -501,9 +501,9 @@ static int LZ4HC_compress_generic ( if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = 256; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 256); + case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = 1<<14; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM); + case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -575,13 +575,15 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, @@ -641,6 +643,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit; } return dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 59d912e..a3736f8 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -153,6 +153,7 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t nextToUpdateBT; /* index from which to continue binary tree update */ uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -170,6 +171,7 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int nextToUpdateBT; /* index from which to continue binary tree update */ unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; diff --git a/lib/lz4opt.h b/lib/lz4opt.h index c466677..0844f8e 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -106,7 +106,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (ip + MINMATCH > iHighLimit) return 1; - /* First Match */ + /* HC4 match finder */ HashPos = &HashTable[LZ4HC_hashPtr(ip)]; matchIndex = *HashPos; *HashPos = current; @@ -162,7 +162,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; - // if (best_mlen > 8) return best_mlen-8; + /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; return 1; } @@ -172,24 +172,24 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; + U32 idx = ctx->nextToUpdateBT; while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); -} +} /** Tree updater, providing best match */ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit, - size_t best_mlen, LZ4HC_match_t* matches) + size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) { int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ - LZ4HC_updateBinTree(ctx, ip, iHighLimit); + if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ + if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdate = (U32)(ip - ctx->base) + best_mlen; + ctx->nextToUpdateBT = (U32)(ip - ctx->base) + best_mlen; return mnum; } @@ -212,7 +212,8 @@ static int LZ4HC_compress_optimal ( int inputSize, int maxOutputSize, limitedOutput_directive limit, - const size_t sufficient_len + const size_t sufficient_len, + const int fullUpdate ) { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; @@ -238,7 +239,7 @@ static int LZ4HC_compress_optimal ( memset(opt, 0, sizeof(LZ4HC_optimal_t)); last_pos = 0; llen = ip - anchor; - match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); @@ -294,7 +295,7 @@ static int LZ4HC_compress_optimal ( if (cur == last_pos || inr >= mflimit) break; LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); - match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches); + match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { -- cgit v0.12 From 4f050b68d817776e3c1fac6c449b5f2b2adfbcb9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 9 Dec 2016 18:17:46 +0100 Subject: tree update dependent from compression level --- lib/lz4hc.c | 6 ++++-- lib/lz4hc.h | 1 + lib/lz4opt.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e1eadf7..44a1340 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -574,8 +574,10 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ - LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + if (ctxPtr->compressionLevel >= LZ4HC_MIN_CLEVEL_OPT) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index a3736f8..7ce5958 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -46,6 +46,7 @@ extern "C" { /* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 +#define LZ4HC_MIN_CLEVEL_OPT 11 #define LZ4HC_MAX_CLEVEL 12 diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 0844f8e..f487edf 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -36,7 +36,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) -#include + #define LZ4_OPT_NUM (1<<12) @@ -189,7 +189,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdateBT = (U32)(ip - ctx->base) + best_mlen; + ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen); return mnum; } -- cgit v0.12 From 913b98fd6fe814ee4c719a90a0642497c9d3d0dc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Dec 2016 22:13:36 +0100 Subject: fix `make` concurrency build (#277) --- Makefile | 4 +++- NEWS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 48e6752..c77f697 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,9 @@ endif .PHONY: default all lib lz4 clean test versionsTest examples -default: lib lz4-release +default: + @$(MAKE) -C $(LZ4DIR) + @$(MAKE) -C $(PRGDIR) all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/NEWS b/NEWS index 292cd27..ff62ca8 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v1.7.5 cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) doc : markdown version of man page, by Takayuki Matsuoka (#279) +build : Makefile : fix concurrency lib+exe build (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12 From 45b14ab8e0584adb97cb8d9366687f623bdd9123 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:03:13 +0100 Subject: added platfom.h --- programs/platform.h | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 programs/platform.h diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 0000000..562c831 --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ +#endif +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ + || defined __x86_64__ || defined _M_X64 /* x86 64-bit */ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ || + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */) +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ************************************** +* Unix Large Files support (>4GB) +****************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + + + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) + /* UNIX-style OS. ------------------------------------------- */ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif + +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ -- cgit v0.12 From bb69cdf2583a56c6c703f5b91aae0632912f5a8a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:03:30 +0100 Subject: updated util.h --- programs/util.h | 128 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/programs/util.h b/programs/util.h index 1ad61bc..7bcc4c4 100644 --- a/programs/util.h +++ b/programs/util.h @@ -20,6 +20,7 @@ - LZ4 source repository : https://github.com/lz4/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -27,70 +28,64 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif - /*-**************************************** * Dependencies ******************************************/ -#include /* features.h with _POSIX_C_SOURCE, malloc */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include /* malloc */ #include /* fprintf */ -#include /* strerr, strlen, memcpy */ -#include /* ptrdiff_t */ #include /* stat, utime */ #include /* stat */ #if defined(_MSC_VER) - #include /* utime */ - #include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else - #include /* chown, stat */ - #include /* utime */ +# include /* chown, stat */ +# include /* utime */ #endif #include /* time */ #include -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) +/*-************************************ +* OS-specific Includes +**************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif #else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) +# define fseek fseeko /* fseeko() added in FreeBSD 3.2 */ +# endif +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) #endif @@ -102,7 +97,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -112,7 +107,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -124,6 +119,12 @@ extern "C" { #endif +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + /*-************************************************************** * Basic Types *****************************************************************/ @@ -148,6 +149,20 @@ extern "C" { /*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline +#else +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** * Time functions ******************************************/ #if !defined(_WIN32) @@ -362,8 +377,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(__APPLE__) && defined(__MACH__)) || \ - ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ @@ -427,7 +441,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } @@ -443,16 +457,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -474,8 +487,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i Date: Tue, 20 Dec 2016 18:05:43 +0100 Subject: executables use platform.h --- programs/bench.c | 5 +++-- programs/lz4cli.c | 24 ++---------------------- programs/lz4io.c | 33 ++------------------------------- tests/fullbench.c | 16 +--------------- 4 files changed, 8 insertions(+), 70 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 434da8b..f077b42 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -27,10 +27,11 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ -#include /* fprintf, fopen, ftello64 */ +#include /* fprintf, fopen, ftello */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include "datagen.h" /* RDG_genBuffer */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 5bd06d9..e552de7 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -39,17 +39,11 @@ /* #define ENABLE_LZ4C_LEGACY_OPTIONS */ -/************************************** -* Compiler Options -***************************************/ -/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */ -#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within on unix */ - - /**************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST */ #include /* fprintf, getchar */ #include /* exit, calloc, free */ #include /* strcmp, strlen */ @@ -59,20 +53,6 @@ #include "lz4.h" /* LZ4_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /***************************** * Constants ******************************/ diff --git a/programs/lz4io.c b/programs/lz4io.c index 46daaf7..a9e5515 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -33,10 +33,6 @@ /************************************** * Compiler Options **************************************/ -#define _LARGE_FILES /* Large file support on 32-bits AIX */ -#define _FILE_OFFSET_BITS 64 /* off_t width */ -#define _LARGEFILE_SOURCE - #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif @@ -44,7 +40,8 @@ /***************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_getFileStat */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_getFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ #include /* strcmp, strlen */ @@ -57,32 +54,6 @@ #include "lz4frame.h" -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - /***************************** * Constants diff --git a/tests/fullbench.c b/tests/fullbench.c index 7d54492..df6d4fc 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -23,20 +23,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif // S_ISREG & gettimeofday() are not supported by MSVC #if defined(_MSC_VER) || defined(_WIN32) @@ -48,7 +34,7 @@ * Includes **************************************/ #include /* malloc, free */ -#include /* fprintf, fopen, ftello64 */ +#include /* fprintf, fopen, ftello */ #include /* stat64 */ #include /* stat64 */ #include /* strcmp */ -- cgit v0.12 From 6adf05d1ed83053b8bb1f762494d2c10fdd8ac1d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 18:56:09 +0100 Subject: improved platform.h --- programs/datagen.c | 13 +------ programs/lz4cli.c | 4 +-- programs/lz4io.c | 4 +-- programs/platform.h | 100 +++++++++++++++++++++++++++++++++++----------------- programs/util.h | 41 --------------------- 5 files changed, 72 insertions(+), 90 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index f97c2d0..f44e310 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,6 +26,7 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ #include /* malloc */ #include /* FILE, fwrite */ #include /* memcpy */ @@ -51,18 +52,6 @@ /************************************** -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -#else -# define SET_BINARY_MODE(file) -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index e552de7..0b0fbd4 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -42,8 +42,8 @@ /**************************** * Includes *****************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ -#include "util.h" /* UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* fprintf, getchar */ #include /* exit, calloc, free */ #include /* strcmp, strlen */ diff --git a/programs/lz4io.c b/programs/lz4io.c index a9e5515..506d661 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -40,8 +40,8 @@ /***************************** * Includes *****************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_getFileStat */ +#include "platform.h" /* Compiler options, Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ #include /* strcmp, strlen */ diff --git a/programs/platform.h b/programs/platform.h index 562c831..df70037 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -24,9 +24,53 @@ extern "C" { # define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif +# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 #endif @@ -48,51 +92,41 @@ extern "C" { #endif -/* ************************************** -* Unix Large Files support (>4GB) -****************************************/ -#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ # define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ # endif # if defined(_AIX) || defined(__hpux) # define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ # endif +# define fseek fseeko #endif - - - -/* ************************************************************ -* Detect POSIX version -* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX -* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION -***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) - /* UNIX-style OS. ------------------------------------------- */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ -# define PLATFORM_POSIX_VERSION 200112L +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # else -# if defined(__linux__) || defined(__linux) -# define _POSIX_C_SOURCE 200112L /* use feature test macro */ -# endif -# include /* declares _POSIX_VERSION */ -# if defined(_POSIX_VERSION) /* POSIX compliant */ -# define PLATFORM_POSIX_VERSION _POSIX_VERSION -# else -# define PLATFORM_POSIX_VERSION 0 -# endif +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) # endif -#endif - -#if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION -1 +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) #endif diff --git a/programs/util.h b/programs/util.h index 7bcc4c4..5ec1a4a 100644 --- a/programs/util.h +++ b/programs/util.h @@ -48,47 +48,6 @@ extern "C" { #include -/*-************************************ -* OS-specific Includes -**************************************/ -#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko /* fseeko() added in FreeBSD 3.2 */ -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - - /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ -- cgit v0.12 From 2efedaf7dc8f20d2e5761ee600712bdb0371bc5c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 19:31:05 +0100 Subject: fixed gcc warnings --- programs/platform.h | 2 +- programs/util.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/platform.h b/programs/platform.h index df70037..244b2d8 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -116,9 +116,9 @@ extern "C" { # include /* _O_BINARY */ # include /* _setmode, _fileno, _get_osfhandle */ # if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } # include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } # define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } # else # define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) diff --git a/programs/util.h b/programs/util.h index 5ec1a4a..b0529b0 100644 --- a/programs/util.h +++ b/programs/util.h @@ -34,6 +34,7 @@ extern "C" { ******************************************/ #include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ #include /* malloc */ +#include /* size_t, ptrdiff_t */ #include /* fprintf */ #include /* stat, utime */ #include /* stat */ @@ -339,6 +340,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ +# include /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { -- cgit v0.12 From 5f0cc8e2d0dc473185ebac61b03fb6a4262797fb Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 20:52:18 +0100 Subject: fixed gcc warnings (2) --- programs/lz4io.c | 4 +++- programs/platform.h | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 506d661..a69b80f 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -53,7 +53,9 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" - +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko +#endif /***************************** * Constants diff --git a/programs/platform.h b/programs/platform.h index 244b2d8..e983756 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -41,7 +41,7 @@ extern "C" { ***************************************************************/ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ # define PLATFORM_POSIX_VERSION 200112L # else @@ -105,7 +105,6 @@ extern "C" { # if defined(_AIX) || defined(__hpux) # define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ # endif -# define fseek fseeko #endif -- cgit v0.12 From e1332241a586cf95476fc19b2f6ba14b7b685863 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 21:11:39 +0100 Subject: improved formatting --- programs/platform.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index e983756..7e004ca 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -39,10 +39,10 @@ extern "C" { * PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX * PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION ***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__midipix__) || defined(__VMS)) /* UNIX-like OS */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) @@ -78,14 +78,14 @@ extern "C" { * Detect 64-bit OS * http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros ****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ - || defined __x86_64__ || defined _M_X64 /* x86 64-bit */ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ || - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */) +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ # if !defined(__64BIT__) # define __64BIT__ 1 # endif -- cgit v0.12 From 090cb887a3fb2b3974b1d0fd228ae08ddc6a2329 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 22:21:03 +0100 Subject: improved formatting (2) --- programs/lz4io.c | 2 ++ programs/platform.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index a69b80f..15c69f0 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -53,9 +53,11 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" +#if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif +#endif /***************************** * Constants diff --git a/programs/platform.h b/programs/platform.h index 7e004ca..ae9587e 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -84,7 +84,7 @@ extern "C" { || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || _ADDR64 /* Cray */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ # if !defined(__64BIT__) # define __64BIT__ 1 -- cgit v0.12 From 452cc970d6b35601a2501dab47050592c97bc4ed Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 23:38:32 +0100 Subject: reordered definitons in platform.h --- programs/lz4cli.c | 9 +++++++ programs/platform.h | 69 +++++++++++++++++++++++++++-------------------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0b0fbd4..366aed5 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -498,6 +498,15 @@ int main(int argc, const char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { diff --git a/programs/platform.h b/programs/platform.h index ae9587e..b8911f2 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -14,6 +14,7 @@ extern "C" { #endif + /* ************************************** * Compiler Options ****************************************/ @@ -33,6 +34,40 @@ extern "C" { #endif +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + /* ************************************************************ * Detect POSIX version * PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows @@ -74,40 +109,6 @@ extern "C" { #endif -/* ************************************** -* Detect 64-bit OS -* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros -****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ - || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ -# if !defined(__64BIT__) -# define __64BIT__ 1 -# endif -#endif - - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - - /****************************** * OS-specific Includes ******************************/ -- cgit v0.12 From 9546ba62d01a9618aab91eafe77929120653d275 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:44:59 +0100 Subject: executables use basic types from util.h --- programs/datagen.c | 20 +------------------- programs/platform.h | 26 ++++++++++++++++++-------- programs/util.h | 11 +++++------ tests/Makefile | 2 +- tests/datagencli.c | 20 +------------------- tests/frametest.c | 22 ++-------------------- tests/fullbench.c | 46 ++-------------------------------------------- tests/fuzzer.c | 24 ++---------------------- 8 files changed, 32 insertions(+), 139 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index f44e310..a61afc0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -27,31 +27,13 @@ * Includes **************************************/ #include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ #include /* malloc */ #include /* FILE, fwrite */ #include /* memcpy */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/platform.h b/programs/platform.h index b8911f2..c4b97c8 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,11 +1,21 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ #ifndef PLATFORM_H_MODULE #define PLATFORM_H_MODULE diff --git a/programs/util.h b/programs/util.h index b0529b0..409f6d7 100644 --- a/programs/util.h +++ b/programs/util.h @@ -15,10 +15,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #ifndef UTIL_H_MODULE @@ -88,7 +84,9 @@ extern "C" { /*-************************************************************** * Basic Types *****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#ifndef BASIC_TYPES_DEFINED +#define BASIC_TYPES_DEFINED +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -105,7 +103,8 @@ extern "C" { typedef signed int S32; typedef unsigned long long U64; typedef signed long long S64; -#endif +#endif +#endif /*-**************************************** diff --git a/tests/Makefile b/tests/Makefile index 01b45bc..3e39608 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -43,7 +43,7 @@ CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-e -Wdeclaration-after-statement -Wstrict-prototypes \ -Wpointer-arith -Wstrict-aliasing=1 CFLAGS += $(MOREFLAGS) -CPPFLAGS:= -I$(LIBDIR) -DXXH_NAMESPACE=LZ4_ +CPPFLAGS:= -I$(LIBDIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_ FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) diff --git a/tests/datagencli.c b/tests/datagencli.c index 0b78d37..c985197 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -27,31 +27,13 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32 */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ #include "lz4.h" /* LZ4_VERSION_STRING */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..ba971d6 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -27,7 +27,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -35,6 +34,8 @@ /*-************************************ * Includes **************************************/ +#include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include /* malloc, free */ #include /* fprintf */ #include /* strcmp */ @@ -45,25 +46,6 @@ #include "xxhash.h" /* XXH64 */ -/*-************************************ -* Basic Types -**************************************/ -#if !defined(__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - /* unoptimized version; solves endianess & alignment issues */ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) { diff --git a/tests/fullbench.c b/tests/fullbench.c index df6d4fc..8d55d2d 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -33,6 +33,7 @@ /************************************** * Includes **************************************/ +#include "util.h" /* U32, UTIL_getFileSize */ #include /* malloc, free */ #include /* fprintf, fopen, ftello */ #include /* stat64 */ @@ -48,34 +49,6 @@ /************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** * Constants **************************************/ #define PROGRAM_DESCRIPTION "LZ4 speed analyzer" @@ -180,21 +153,6 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - /********************************************************* * Benchmark function *********************************************************/ @@ -384,7 +342,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory size adjustments */ - inFileSize = BMK_GetFileSize(inFileName); + inFileSize = UTIL_getFileSize(inFileName); if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 260208a..18ba7c8 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,8 +27,6 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif @@ -37,6 +35,8 @@ /*-************************************ * Dependencies **************************************/ +#include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include #include /* fgets, sscanf */ #include /* strcmp */ @@ -47,26 +47,6 @@ /*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ -#endif - - -/*-************************************ * Constants **************************************/ #define NB_ATTEMPTS (1<<16) -- cgit v0.12 From 64cbc4e1dc17492d89915a9d774dc15ee438ccfe Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 10:22:40 +0100 Subject: improved MinGW support --- programs/lz4cli.c | 3 +++ programs/lz4io.c | 6 ------ programs/platform.h | 3 +++ tests/frametest.c | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 366aed5..e03aa98 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -507,6 +507,9 @@ int main(int argc, const char** argv) #ifdef PLATFORM_POSIX_VERSION DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif +#ifdef _FILE_OFFSET_BITS + DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { diff --git a/programs/lz4io.c b/programs/lz4io.c index 15c69f0..df3ed41 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -30,12 +30,6 @@ - The license of this source file is GPLv2. */ -/************************************** -* Compiler Options -**************************************/ -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif /***************************** * Includes diff --git a/programs/platform.h b/programs/platform.h index c4b97c8..02ebd4c 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -42,6 +42,9 @@ extern "C" { # define fseek _fseeki64 # endif #endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ +#endif /* ************************************** diff --git a/tests/frametest.c b/tests/frametest.c index ba971d6..8b7caba 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -92,7 +92,7 @@ static clock_t g_clockTime = 0; *****************************************/ static U32 no_prompt = 0; static U32 displayLevel = 2; -static U32 pause = 0; +static U32 use_pause = 0; /*-******************************************************* @@ -705,7 +705,7 @@ _end: free(compressedBuffer); free(decodedBuffer); - if (pause) { + if (use_pause) { DISPLAY("press enter to finish \n"); (void)getchar(); } @@ -777,7 +777,7 @@ int main(int argc, const char** argv) break; case 'p': /* pause at the end */ argument++; - pause = 1; + use_pause = 1; break; case 'i': -- cgit v0.12 From 3c6ce9c66526f0f6e3c200ddb64db2c66be5e031 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 10:39:27 +0100 Subject: fixed gcc 4.4 support --- tests/fuzzer.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 18ba7c8..10e9139 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -47,6 +47,14 @@ /*-************************************ +* Basic Types +**************************************/ +#if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */ +#endif + + +/*-************************************ * Constants **************************************/ #define NB_ATTEMPTS (1<<16) @@ -980,7 +988,7 @@ int main(int argc, const char** argv) int nbTests = NB_ATTEMPTS; int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; + int use_pause = 0; const char* programName = argv[0]; U32 duration = 0; @@ -992,7 +1000,7 @@ int main(int argc, const char** argv) // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { pause=0; seedset=1; g_displayLevel=1; continue; } + if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; } argument++; while (*argument!=0) { @@ -1008,7 +1016,7 @@ int main(int argc, const char** argv) case 'p': /* pause at the end */ argument++; - pause=1; + use_pause=1; break; case 'i': @@ -1098,7 +1106,7 @@ int main(int argc, const char** argv) if (nbTests<=0) nbTests=1; { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { + if (use_pause) { DISPLAY("press enter ... \n"); (void)getchar(); } -- cgit v0.12 From 2fd7eb554a17ce4815c60b29a226130e0ea448d6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:53:16 +0100 Subject: fixed Visual Studio compilation --- programs/lz4io.c | 8 ++++++++ programs/platform.h | 3 --- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index df3ed41..ca3cea5 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -47,12 +47,20 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +#endif + #if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif #endif + /***************************** * Constants *****************************/ diff --git a/programs/platform.h b/programs/platform.h index 02ebd4c..fe4bfde 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -38,9 +38,6 @@ extern "C" { # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif -# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ -- cgit v0.12 From 287f7d3d6d32e48a711ec8979e30a113469e235b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:58:50 +0100 Subject: added IncludePath --- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 363c5ae..76d12c9 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index 8d0a623..e2d95c9 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 6672aaa..85d6c9b 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true -- cgit v0.12 From 253ac12cb76c5bacab69fdc723982ae9a92fcf53 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 12:06:36 +0100 Subject: VS projects: CharacterSet changed to MultiByte --- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench-dll/fullbench-dll.vcxproj | 16 ++++++++-------- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 76d12c9..1998e89 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 16a4f0d..1ee263d 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte @@ -66,20 +66,20 @@ true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); true diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index e2d95c9..f3734dd 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 85d6c9b..46a7c0f 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte -- cgit v0.12 From 4f61505cc40cb8b2c6c371600a427fd685c656a2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 12:09:58 +0100 Subject: datagen.vcxproj: CharacterSet is MultiByte --- visual/VS2010/datagen/datagen.vcxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index aaf81ad..825bef6 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -29,24 +29,24 @@ Application true - Unicode + MultiByte Application true - Unicode + MultiByte Application false true - Unicode + MultiByte Application false true - Unicode + MultiByte -- cgit v0.12 From 385cb4f539134b3cb855a29eef99a4a876f728d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 13:18:02 +0100 Subject: minor update Makefile --- Makefile | 1 + lib/Makefile | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index c77f697..647ab4e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ endif default: @$(MAKE) -C $(LZ4DIR) @$(MAKE) -C $(PRGDIR) + @cp $(PRGDIR)/lz4$(EXT) . all: @$(MAKE) -C $(LZ4DIR) $@ diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..8426330 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -71,6 +71,8 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +LIBLZ4 = liblz4.$(SHARED_EXT_VER) + default: lib-release lib-release: liblz4.a liblz4 @@ -90,18 +92,20 @@ ifeq ($(BUILD_STATIC),yes) @$(AR) rcs $@ *.o endif -liblz4: *.c +$(LIBLZ4): *.c @echo compiling dynamic library $(LIBVER) ifneq (,$(filter Windows%,$(OS))) @$(CC) $(FLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o dll\$@.dll dlltool -D dll\liblz4.dll -d dll\liblz4.def -l dll\liblz4.lib else - @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@ @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + @ln -sf $@ liblz4.$(SHARED_EXT_MAJOR) + @ln -sf $@ liblz4.$(SHARED_EXT) endif +liblz4: $(LIBLZ4) + clean: @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib @echo Cleaning library completed @@ -123,21 +127,21 @@ liblz4.pc: liblz4.pc.in Makefile install: lib liblz4.pc @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @install -m 755 liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) + @install -m 755 liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR) endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR) @echo lz4 static and shared libraries installed uninstall: + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) @$(RM) $(DESTDIR)$(LIBDIR)/liblz4.a @$(RM) $(DESTDIR)$(INCLUDEDIR)/lz4.h -- cgit v0.12 From 7cf0bb97b2a988cb17435780d19e145147dd9f70 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 14:18:01 +0100 Subject: LZ4F_compressBound(0) provides upper bound for LZ4F_flush() and LZ4F_compressEnd() [#290, suggested by @vtermanis] --- lib/lz4frame.c | 15 +++++++++++---- lib/lz4frame.h | 23 ++++++++++++----------- tests/frametest.c | 7 +++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 3c2b788..626ac98 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; /* max Frame Header Size */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */ static const size_t BHSize = 4; @@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +/* LZ4F_compressBound() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; size_t const blockSize = LZ4F_getBlockSize(bid); size_t const maxBuffered = blockSize - 1; size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const maxSrcSize = srcSize + bufferedSize; unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); - size_t const partialBlockSize = srcSize & (blockSize-1); - size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */ + size_t const lastBlockSize = flush ? partialBlockSize : 0; unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ @@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 3104d2e..a4a4ce6 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); /* Compression */ +#define LZ4F_HEADER_SIZE_MAX 15 LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. + * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * @return : number of bytes written into dstBuffer for the header * or an error code (which can be tested using LZ4F_isError()) @@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. + * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. */ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. - * This value is provided by using LZ4F_compressBound(). + * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) */ @@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci * or an error code if it fails (which can be tested using LZ4F_isError()) */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressEnd() : - * To properly finish the compressed frame, invoke LZ4F_compressEnd(). + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). * It will flush whatever data remained within `cctx` (like LZ4_flush()) * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * or an error code if it fails (which can be tested using LZ4F_isError()) * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. diff --git a/tests/frametest.c b/tests/frametest.c index a99728f..fd07377 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -204,6 +204,13 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + /* LZ4F_compressBound() : special case : srcSize == 0 */ + DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); + { size_t const cBound = LZ4F_compressBound(0, NULL); + if (cBound < 64 KB) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (U32)cBound); + } + /* Special case : null-content frame */ testSize = 0; DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n"); -- cgit v0.12 From bc0839c5df185129e899640e77a05f1c8fa595f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:00:43 +0100 Subject: minor fix for travis-install test --- Makefile | 4 ++-- NEWS | 5 +++-- contrib/cmake_unofficial/.gitignore | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 647ab4e..1432e6b 100644 --- a/Makefile +++ b/Makefile @@ -99,10 +99,10 @@ uninstall: @$(MAKE) -C $(PRGDIR) $@ travis-install: - $(MAKE) install PREFIX=~/install_test_dir + $(MAKE) -j1 install PREFIX=~/install_test_dir test: - $(MAKE) -C $(TESTDIR) test + $(MAKE) -C $(TESTDIR) $@ clangtest: clean clang -v diff --git a/NEWS b/NEWS index ff62ca8..e2f7666 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.7.5 -cli : fix minor notification when using -r recursive mode lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) +cli : fix minor notification when using -r recursive mode +API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix concurrency lib+exe build (#277) +build : Makefile : fix make lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/contrib/cmake_unofficial/.gitignore b/contrib/cmake_unofficial/.gitignore index 0f81929..d39505d 100644 --- a/contrib/cmake_unofficial/.gitignore +++ b/contrib/cmake_unofficial/.gitignore @@ -5,3 +5,5 @@ CMakeFiles *.cmake Makefile liblz4.pc +lz4c +install_manifest.txt -- cgit v0.12 From 70526a11e033a640fb9e7ae80fe75b465d484bf0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 15:33:53 +0100 Subject: fixed lib/clean --- lib/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index 8426330..701dc07 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -107,7 +107,8 @@ endif liblz4: $(LIBLZ4) clean: - @$(RM) core *.o *.a *.$(SHARED_EXT) liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) core *.o liblz4.pc dll/liblz4.dll dll/liblz4.lib + @$(RM) *.a *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) @echo Cleaning library completed -- cgit v0.12 From 973bc79740f571c2b6700cab16a319df9aed2a05 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 16:10:09 +0100 Subject: util.h and platform.h based on zstd --- programs/lz4io.c | 6 ++++- programs/platform.h | 13 +++------- programs/util.h | 72 +++++++++++++++++++++++++++-------------------------- tests/frametest.c | 2 +- tests/fullbench.c | 1 + tests/fuzzer.c | 3 ++- 6 files changed, 50 insertions(+), 47 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index ca3cea5..36498b2 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -31,10 +31,14 @@ */ +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + /***************************** * Includes *****************************/ -#include "platform.h" /* Compiler options, Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ diff --git a/programs/platform.h b/programs/platform.h index fe4bfde..f1040c0 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -25,23 +25,17 @@ extern "C" { #endif + /* ************************************** * Compiler Options ****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif #if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif #endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ -#endif /* ************************************** @@ -110,6 +104,7 @@ extern "C" { * Detect if isatty() and fileno() are available ************************************************/ #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ diff --git a/programs/util.h b/programs/util.h index 409f6d7..c670c20 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,26 +25,51 @@ extern "C" { #endif + /*-**************************************** * Dependencies ******************************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ -#include /* malloc */ -#include /* size_t, ptrdiff_t */ -#include /* fprintf */ -#include /* stat, utime */ -#include /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include /* malloc */ +#include /* size_t, ptrdiff_t */ +#include /* fprintf */ +#include /* stat, utime */ +#include /* stat */ #if defined(_MSC_VER) -# include /* utime */ -# include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else # include /* chown, stat */ # include /* utime */ #endif -#include /* time */ +#include /* time */ #include + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ @@ -81,35 +106,12 @@ extern "C" { #define LIST_SIZE_INCREASE (8*1024) -/*-************************************************************** -* Basic Types -*****************************************************************/ -#ifndef BASIC_TYPES_DEFINED -#define BASIC_TYPES_DEFINED -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif -#endif - - /*-**************************************** * Compiler specifics ******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif #if defined(__GNUC__) # define UTIL_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) diff --git a/tests/frametest.c b/tests/frametest.c index 8b7caba..869be76 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -27,6 +27,7 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -34,7 +35,6 @@ /*-************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ #include "util.h" /* U32 */ #include /* malloc, free */ #include /* fprintf */ diff --git a/tests/fullbench.c b/tests/fullbench.c index 8d55d2d..f489392 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -33,6 +33,7 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */ #include "util.h" /* U32, UTIL_getFileSize */ #include /* malloc, free */ #include /* fprintf, fopen, ftello */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 10e9139..ff02e0c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,6 +27,7 @@ * Compiler options **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ #endif @@ -35,7 +36,7 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options */ +#include "platform.h" /* _CRT_SECURE_NO_WARNINGS */ #include "util.h" /* U32 */ #include #include /* fgets, sscanf */ -- cgit v0.12 From f084b653835940c749d4fc92a610e861a6644aa2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 17:13:38 +0100 Subject: test Large File support for Mac OS-X in 32-bits mode --- .travis.yml | 4 +++- programs/bench.c | 8 ++++++++ programs/lz4io.c | 13 +++++++++++-- tests/Makefile | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3f476b..dc61505 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ matrix: include: # OS X Mavericks - os: osx - env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + install: + - export CC=clang + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" && CFLAGS=-m32 make -C tests clean test-lz4-contentSize' COMPILER=clang # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux diff --git a/programs/bench.c b/programs/bench.c index f077b42..01e610c 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -24,6 +24,14 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ diff --git a/programs/lz4io.c b/programs/lz4io.c index 36498b2..3d69a6e 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -31,10 +31,17 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif + /***************************** * Includes *****************************/ @@ -644,8 +651,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer storedSkips += (unsigned)(nb0T * sizeT); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + errno = 0; + { int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; diff --git a/tests/Makefile b/tests/Makefile index 3e39608..b63493d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -173,7 +173,7 @@ test-lz4-contentSize: lz4 datagen $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 diff -s tmp tmp2 # test large size [2-4] GB - @./datagen -g3G -P100 | $(LZ4) --verbose | $(LZ4) --decompress --force --sparse - tmp + @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp @ls -ls tmp @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 @ls -ls tmp2 -- cgit v0.12 From 58124506358648b05477698e9136d7f651f11e01 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 18:46:14 +0100 Subject: use fseeko for 32-bit MacOS --- programs/lz4io.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 3d69a6e..877c034 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -58,18 +58,16 @@ #include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" + /* ************************************** * Compiler Options ****************************************/ #if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ # define fseek _fseeki64 #endif - -#if 0 #if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ # define fseek fseeko #endif -#endif /***************************** -- cgit v0.12 From e6af952f8495de3a9e3cabb9a21e0ba4d7d28ff0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 19:49:06 +0100 Subject: improved comments --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 877c034..e8370ea 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -45,7 +45,7 @@ /***************************** * Includes *****************************/ -#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ -- cgit v0.12 From fea95c156761ee10d32a4f12105ebd90c67d4f47 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 22 Dec 2016 10:58:58 +0100 Subject: use FindFirstFileA and FindNextFileA on Windows --- programs/lz4io.c | 2 +- programs/util.h | 6 +++--- visual/VS2010/datagen/datagen.vcxproj | 8 ++++---- visual/VS2010/frametest/frametest.vcxproj | 8 ++++---- visual/VS2010/fullbench-dll/fullbench-dll.vcxproj | 8 ++++---- visual/VS2010/fullbench/fullbench.vcxproj | 8 ++++---- visual/VS2010/fuzzer/fuzzer.vcxproj | 8 ++++---- visual/VS2010/lz4/lz4.vcxproj | 8 ++++---- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index e8370ea..640c76d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -49,7 +49,7 @@ #include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include /* malloc, free */ -#include /* strcmp, strlen */ +#include /* strerror, strcmp, strlen */ #include /* clock */ #include /* stat64 */ #include /* stat64 */ diff --git a/programs/util.h b/programs/util.h index c670c20..f3ff1b2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -283,7 +283,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -295,7 +295,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -332,7 +332,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; diff --git a/visual/VS2010/datagen/datagen.vcxproj b/visual/VS2010/datagen/datagen.vcxproj index 825bef6..aaf81ad 100644 --- a/visual/VS2010/datagen/datagen.vcxproj +++ b/visual/VS2010/datagen/datagen.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/frametest/frametest.vcxproj b/visual/VS2010/frametest/frametest.vcxproj index 1998e89..76d12c9 100644 --- a/visual/VS2010/frametest/frametest.vcxproj +++ b/visual/VS2010/frametest/frametest.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj index 1ee263d..c10552a 100644 --- a/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/visual/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fullbench/fullbench.vcxproj b/visual/VS2010/fullbench/fullbench.vcxproj index f3734dd..e2d95c9 100644 --- a/visual/VS2010/fullbench/fullbench.vcxproj +++ b/visual/VS2010/fullbench/fullbench.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/fuzzer/fuzzer.vcxproj b/visual/VS2010/fuzzer/fuzzer.vcxproj index 46a7c0f..85d6c9b 100644 --- a/visual/VS2010/fuzzer/fuzzer.vcxproj +++ b/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode diff --git a/visual/VS2010/lz4/lz4.vcxproj b/visual/VS2010/lz4/lz4.vcxproj index 295a94b..693e121 100644 --- a/visual/VS2010/lz4/lz4.vcxproj +++ b/visual/VS2010/lz4/lz4.vcxproj @@ -29,24 +29,24 @@ Application true - MultiByte + Unicode Application true - MultiByte + Unicode Application false true - MultiByte + Unicode Application false true - MultiByte + Unicode -- cgit v0.12 From 52cac9a97342641315c76cfb861206d6acd631a8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:41:05 +0100 Subject: updated a few macros names --- lib/lz4frame.c | 14 +++++++------- lib/lz4hc.c | 4 ++-- lib/lz4hc.h | 8 ++++---- programs/bench.c | 6 +++--- programs/lz4cli.c | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 626ac98..a0a625b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -329,7 +329,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (prefs.frameInfo.contentSize != 0) prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { + if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; } @@ -356,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu if (LZ4F_isError(tailSize)) return tailSize; dstPtr += tailSize; } - if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ + if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -424,10 +424,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->prefs = *preferencesPtr; /* ctx Management */ - { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -452,7 +452,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -533,7 +533,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { - if (level < LZ4HC_MIN_CLEVEL) { + if (level < LZ4HC_CLEVEL_MIN) { if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } @@ -543,7 +543,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { - if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 44a1340..ae245b5 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -497,7 +497,7 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL; + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); @@ -574,7 +574,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - if (ctxPtr->compressionLevel >= LZ4HC_MIN_CLEVEL_OPT) + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 7ce5958..ad731d9 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -44,10 +44,10 @@ extern "C" { /* --- Useful constants --- */ -#define LZ4HC_MIN_CLEVEL 3 -#define LZ4HC_DEFAULT_CLEVEL 9 -#define LZ4HC_MIN_CLEVEL_OPT 11 -#define LZ4HC_MAX_CLEVEL 12 +#define LZ4HC_CLEVEL_MIN 3 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 11 +#define LZ4HC_CLEVEL_MAX 12 /*-************************************ diff --git a/programs/bench.c b/programs/bench.c index 434da8b..406adcf 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -170,7 +170,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -494,8 +494,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 5bd06d9..80d84a1 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -208,7 +208,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); -- cgit v0.12 From 7eb16d97d238e2b98f1e6453ed6ad64058cb99c1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 11:50:21 +0100 Subject: updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index e2f7666..bb77e04 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.7.5 +lz4hc : new high compression mode : levels 10-12 compress more and slower, by @inikep. lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) -- cgit v0.12 From 9b4b081c151ddde70587f0838062a9fd660bcc49 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 22 Dec 2016 17:18:39 +0100 Subject: "make test" is now compatible with Solaris --- programs/Makefile | 6 +++--- tests/Makefile | 38 ++++++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 74a2440..5f14ad7 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -108,9 +108,9 @@ preview-man: clean-man man #------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#make install is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and +#Solaris targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) unlz4: lz4 ln -s lz4 unlz4 diff --git a/tests/Makefile b/tests/Makefile index b63493d..d23ed46 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -120,15 +120,21 @@ versionsTest: #------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and +#Solaris targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) MD5:=md5 -r endif +DIFF:=diff +ifneq (,$(filter $(shell uname),SunOS)) +DIFF:=gdiff +endif + + test: test-lz4 test-lz4c test-fasttest test-frametest test-fullbench test-fuzzer test32: CFLAGS+=-m32 @@ -138,18 +144,18 @@ test-lz4-sparse: lz4 datagen @echo "\n ---- test sparse file support ----" ./datagen -g5M -P100 > tmpSrc $(LZ4) -B4D tmpSrc | $(LZ4) -dv --sparse > tmpB4 - diff -s tmpSrc tmpB4 + $(DIFF) -s tmpSrc tmpB4 $(LZ4) -B5D tmpSrc | $(LZ4) -dv --sparse > tmpB5 - diff -s tmpSrc tmpB5 + $(DIFF) -s tmpSrc tmpB5 $(LZ4) -B6D tmpSrc | $(LZ4) -dv --sparse > tmpB6 - diff -s tmpSrc tmpB6 + $(DIFF) -s tmpSrc tmpB6 $(LZ4) -B7D tmpSrc | $(LZ4) -dv --sparse > tmpB7 - diff -s tmpSrc tmpB7 + $(DIFF) -s tmpSrc tmpB7 $(LZ4) tmpSrc | $(LZ4) -dv --no-sparse > tmpNoSparse - diff -s tmpSrc tmpNoSparse + $(DIFF) -s tmpSrc tmpNoSparse ls -ls tmp* ./datagen -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmpOdd # Odd size file (to generate non-full last block) - ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd + ./datagen -s1 -g1200007 -P100 | $(DIFF) -s - tmpOdd ls -ls tmpOdd @$(RM) tmp* @echo "\n Compatibility with Console :" @@ -163,7 +169,7 @@ test-lz4-sparse: lz4 datagen $(LZ4) -d -v tmpC tmpR $(LZ4) -d -v tmpC >> tmpR ls -ls tmp* - diff tmp2M tmpR + $(DIFF) tmp2M tmpR @$(RM) tmp* test-lz4-contentSize: lz4 datagen @@ -171,13 +177,13 @@ test-lz4-contentSize: lz4 datagen ./datagen -g15M > tmp $(LZ4) -v tmp | $(LZ4) -t $(LZ4) -v --content-size tmp | $(LZ4) -d > tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 # test large size [2-4] GB @./datagen -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmp @ls -ls tmp @./datagen -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmp2 @ls -ls tmp2 - diff -s tmp tmp2 + $(DIFF) -s tmp tmp2 @$(RM) tmp* test-lz4-frame-concatenation: lz4 datagen @@ -218,9 +224,9 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat ./datagen -g16KB | $(LZ4) -9 | $(LZ4) -t ./datagen -g20KB > tmpSrc $(LZ4) < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(DIFF) -q tmpSrc tmpRes $(LZ4) --no-frame-crc < tmpSrc | $(LZ4) -d > tmpRes - diff -q tmpSrc tmpRes + $(DIFF) -q tmpSrc tmpRes ./datagen | $(LZ4) | $(LZ4) -t ./datagen -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t ./datagen -g17M | $(LZ4) -9v | $(LZ4) -qt @@ -241,9 +247,9 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat ls -ls tmp.lz4 && false || true # must fail (lz4cat) $(LZ4) tmp # creates tmp.lz4 $(PRGDIR)/lz4cat < tmp.lz4 > tmp3 # checks lz4cat works with stdin (#285) - diff -q tmp tmp3 + $(DIFF) -q tmp tmp3 $(PRGDIR)/lz4cat < tmp > tmp2 # checks lz4cat works with stdin (#285) - diff -q tmp tmp2 + $(DIFF) -q tmp tmp2 @$(RM) tmp* test-lz4-hugefile: lz4 datagen -- cgit v0.12 From 19c0f21b000ababf11cb38e0d6154742f5e1cd83 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Dec 2016 18:02:09 +0100 Subject: updated Makefile : lz4 no longer recompiled when already up-to-date --- NEWS | 2 +- lib/lz4opt.h | 59 ++++++++++++++++++++++++++++--------------------------- programs/Makefile | 29 ++++++++++++++++----------- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/NEWS b/NEWS index bb77e04..d149128 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @be cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) -build : Makefile : fix make lib+exe concurrency (#277) +build : Makefile : fix make -jX lib+exe concurrency (#277) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index f487edf..94b38e0 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -2,7 +2,7 @@ lz4opt.h - Optimal Mode of LZ4 Copyright (C) 2015-2016, Przemyslaw Skibinski Note : this file is intended to be included within lz4hc.c - + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -33,9 +33,9 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_OPT_NUM (1<<12) @@ -56,23 +56,24 @@ typedef struct } LZ4HC_optimal_t; -FORCE_INLINE size_t LZ4HC_GetLiteralsPrice(size_t litlen) +/* price in bits */ +FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) { size_t price = 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); return price; } -FORCE_INLINE size_t LZ4HC_get_price(size_t litlen, size_t mlen) +/* requires mlen >= MINMATCH */ +FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) { size_t price = 16 + 8; /* 16-bit offset + token */ - price += 8*litlen; - if (litlen >= (int)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); + price += LZ4HC_literalsPrice(litlen); mlen -= MINMATCH; - if (mlen >= (int)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); + if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); return price; } @@ -103,7 +104,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( U32 matchIndex; size_t matchLength = 0; U32* HashPos; - + if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -164,7 +165,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (matchNum) *matchNum = mnum; /* if (best_mlen > 8) return best_mlen-8; */ if (!matchNum) return 1; - return 1; + return 1; } @@ -258,7 +259,7 @@ static int LZ4HC_compress_optimal ( LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); while (mlen <= best_mlen) { litlen = 0; - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); SET_PRICE(mlen, mlen, matches[i].off, litlen, price); mlen++; } @@ -274,18 +275,18 @@ static int LZ4HC_compress_optimal ( if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; if (cur != litlen) { - price = opt[cur - litlen].price + LZ4HC_GetLiteralsPrice(litlen); + price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); } else { - price = LZ4HC_GetLiteralsPrice(llen + litlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); } } else { litlen = 1; - price = opt[cur - 1].price + LZ4HC_GetLiteralsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_GetLiteralsPrice(litlen)); + price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen)); } - + mlen = 1; best_mlen = 0; LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); @@ -293,7 +294,7 @@ static int LZ4HC_compress_optimal ( SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); @@ -317,12 +318,12 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur2].litlen; if (cur2 != litlen) - price = opt[cur2 - litlen].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen); else - price = LZ4HC_get_price(llen + litlen, mlen) - LZ4HC_GetLiteralsPrice(llen); + price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); } else { litlen = 0; - price = opt[cur2].price + LZ4HC_get_price(litlen, mlen); + price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); @@ -340,31 +341,31 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); opt[0].mlen = 1; while (1) { mlen = opt[cur].mlen; offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; + opt[cur].mlen = (int)best_mlen; opt[cur].off = (int)best_off; best_mlen = mlen; best_off = offset; if (mlen > cur) break; cur -= mlen; } - + for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -374,7 +375,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); - if (res) return 0; + if (res) return 0; LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); } diff --git a/programs/Makefile b/programs/Makefile index 74a2440..6f4c8c2 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -43,7 +43,9 @@ PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib -VOID := /dev/null + +SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) +OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 @@ -60,33 +62,36 @@ MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +VOID := nul +EXT :=.exe else -EXT = +VOID := /dev/null +EXT := endif default: lz4-release -lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o lz4$(EXT) - all: lz4 lz4c all32: CFLAGS+=-m32 all32: all lz4: CFLAGS += $(DEBUGFLAGS) -lz4: lz4-release +lz4: $(OBJFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lz4-release: DEBUGFLAGS= +lz4-release: lz4 -lz4c: CFLAGS += $(DEBUGFLAGS) -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o +lz4c : CFLAGS += $(DEBUGFLAGS) +lz4c : $(SRCFILES) $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) +lz4c32: $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) -- cgit v0.12 From ea51ad0ceefd0a47df706a9301a8baf9b2629cc0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 23 Dec 2016 00:02:01 +0100 Subject: Solaris: working "make install" --- lib/Makefile | 53 +++++++++++++++++++++++++++++++++++------------------ programs/Makefile | 37 +++++++++++++++++++++++++------------ tests/Makefile | 2 +- 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 2b47fb0..f9fad29 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -42,11 +42,6 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) BUILD_STATIC:= yes -DESTDIR ?= -PREFIX ?= /usr/local -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include - CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 DEBUGFLAGS:=-g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -107,10 +102,30 @@ clean: @echo Cleaning library completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +ifneq (,$(filter $(shell uname),SunOS)) +PREFIX ?= /usr +INSTALL ?= ginstall +else +PREFIX ?= /usr/local +INSTALL ?= install +endif +DESTDIR ?= +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) +PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig +else +PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig +endif + +INSTALL_LIB ?= $(INSTALL) -m 755 +INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile @echo creating pkgconfig @@ -121,17 +136,19 @@ liblz4.pc: liblz4.pc.in Makefile $< >$@ install: lib liblz4.pc - @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - @install -m 644 liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ + @$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/ $(DESTDIR)$(INCLUDEDIR)/ + @$(INSTALL_DATA) liblz4.pc $(DESTDIR)$(PKGCONFIGDIR)/ + @echo Installing libraries ifeq ($(BUILD_STATIC),yes) - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a endif - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @install -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) + @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @echo Installing includes + @$(INSTALL_DATA) -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 static and shared libraries installed uninstall: diff --git a/programs/Makefile b/programs/Makefile index 5f14ad7..f91a326 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,10 +38,6 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 LZ4DIR := ../lib VOID := /dev/null @@ -107,10 +103,10 @@ preview-man: clean-man man man ./lz4.1 -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and -#Solaris targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) unlz4: lz4 ln -s lz4 unlz4 @@ -118,15 +114,32 @@ unlz4: lz4 lz4cat: lz4 ln -s lz4 lz4cat + +ifneq (,$(filter $(shell uname),SunOS)) +PREFIX ?= /usr +INSTALL ?= ginstall +else +PREFIX ?= /usr/local +INSTALL ?= install +endif +DESTDIR ?= +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 + + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c + @$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages - @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @$(INSTALL_MAN) -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 diff --git a/tests/Makefile b/tests/Makefile index d23ed46..1b92718 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,7 +122,7 @@ versionsTest: #------------------------------------------------------------------------ #make test is validated only for Linux, OSX, kFreeBSD, FreeBSD, Hurd and #Solaris targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD SunOS)) +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) MD5:=md5sum ifneq (,$(filter $(shell uname), Darwin )) -- cgit v0.12 From b22e2bed5e44f2cdacb11cf01d4a15a00bcb76ca Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 23 Dec 2016 10:05:41 +0100 Subject: BSD: improved "make install" --- lib/Makefile | 17 +++++++++++------ programs/Makefile | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index f9fad29..13d0482 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -108,15 +108,20 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) ifneq (,$(filter $(shell uname),SunOS)) -PREFIX ?= /usr INSTALL ?= ginstall else -PREFIX ?= /usr/local INSTALL ?= install endif -DESTDIR ?= -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +PREFIX ?= /usr +else +PREFIX ?= /usr/local +endif + +DESTDIR ?= +LIBDIR ?= $(PREFIX)/lib +INCLUDEDIR ?= $(PREFIX)/include ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly)) PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig @@ -124,7 +129,7 @@ else PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif -INSTALL_LIB ?= $(INSTALL) -m 755 +INSTALL_LIB ?= $(INSTALL) -m 755 INSTALL_DATA ?= $(INSTALL) -m 644 liblz4.pc: liblz4.pc.in Makefile diff --git a/programs/Makefile b/programs/Makefile index f91a326..14b042a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -116,19 +116,24 @@ lz4cat: lz4 ifneq (,$(filter $(shell uname),SunOS)) -PREFIX ?= /usr INSTALL ?= ginstall else -PREFIX ?= /usr/local INSTALL ?= install endif + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +PREFIX ?= /usr +else +PREFIX ?= /usr/local +endif + DESTDIR ?= -BINDIR = $(PREFIX)/bin -MANDIR = $(PREFIX)/share/man/man1 +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/share/man/man1 INSTALL_PROGRAM ?= $(INSTALL) -m 755 -INSTALL_SCRIPT ?= $(INSTALL) -m 755 -INSTALL_MAN ?= $(INSTALL) -m 644 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 install: lz4$(EXT) lz4c$(EXT) -- cgit v0.12 From e6536faf337e1c4e202f32bec860e2c87511d7f3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 11:17:35 +0100 Subject: lib\Makefile: fixed INSTALL_DATA --- lib/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 50d4a41..3e21c1b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -156,9 +156,9 @@ endif @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) @echo Installing includes - @$(INSTALL_DATA) -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @$(INSTALL_DATA) -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @$(INSTALL_DATA) -m 644 lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h + @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @$(INSTALL_DATA) lz4frame.h $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 static and shared libraries installed uninstall: -- cgit v0.12 From f4575f4f148dd62894d5d2f0af2e8b1a5fc08ae2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 13:14:04 +0100 Subject: added test-lz4-opt-parser --- tests/Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 1b92718..bf84034 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -277,7 +277,15 @@ test-lz4-testmode: lz4 datagen $(LZ4) -fm file1-dne file2-dne && false || true $(LZ4) -fm file1-dne file2-dne && false || true -test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \ +test-lz4-opt-parser: lz4 datagen + @echo "\n ---- test opt-parser ----" + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen | $(LZ4) -12 | $(LZ4) -t + ./datagen -g1M | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g256MB | $(LZ4) -11vqB4D | $(LZ4) -qt + +test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ test-lz4-hugefile -- cgit v0.12 From 20381a2fce8216b9d844c8da2ba3ecbd31118560 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 15:31:35 +0100 Subject: fixed -BD compression --- lib/lz4opt.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 94b38e0..46797ef 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -133,7 +133,10 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( if (matchLength > best_mlen) { best_mlen = matchLength; if (matches) { - matches[mnum].off = (int)(ip - match); + if (matchIndex >= dictLimit) + matches[mnum].off = (int)(ip - match); + else + matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ matches[mnum].len = (int)matchLength; mnum++; } -- cgit v0.12 From a3d61cf3dff74e286bb731ce6533c9992c49d4d9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 27 Dec 2016 15:38:07 +0100 Subject: improved test-lz4-opt-parser --- tests/Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index bf84034..97fa782 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -279,11 +279,16 @@ test-lz4-testmode: lz4 datagen test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" - ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t - ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t - ./datagen | $(LZ4) -12 | $(LZ4) -t - ./datagen -g1M | $(LZ4) -11B5 | $(LZ4) -t - ./datagen -g256MB | $(LZ4) -11vqB4D | $(LZ4) -qt + ./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t + ./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t + ./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t + ./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t + ./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t + ./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t + ./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt + ./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t + ./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t + ./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \ test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \ -- cgit v0.12 From d57ff6456d2fbf1c644eb07ff0684646bea6808a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 11:27:17 +0100 Subject: changed default PREFIX and MANDIR --- lib/Makefile | 7 +------ programs/Makefile | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 3e21c1b..2d9c8f3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,12 +118,7 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else -PREFIX ?= /usr/local -endif - +PREFIX ?= /usr/local DESTDIR ?= LIBDIR ?= $(PREFIX)/lib INCLUDEDIR ?= $(PREFIX)/include diff --git a/programs/Makefile b/programs/Makefile index 07bec03..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -126,15 +126,15 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else PREFIX ?= /usr/local -endif - DESTDIR ?= BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else MANDIR ?= $(PREFIX)/share/man/man1 +endif INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_SCRIPT ?= $(INSTALL) -m 755 -- cgit v0.12 From 7a73c5c1fe1cf3c95fded86cf5051a70327dcfd1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 11:34:23 +0100 Subject: changed default PREFIX and MANDIR --- lib/Makefile | 7 +------ programs/Makefile | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 3e21c1b..2d9c8f3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,12 +118,7 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else -PREFIX ?= /usr/local -endif - +PREFIX ?= /usr/local DESTDIR ?= LIBDIR ?= $(PREFIX)/lib INCLUDEDIR ?= $(PREFIX)/include diff --git a/programs/Makefile b/programs/Makefile index 07bec03..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -126,15 +126,15 @@ else INSTALL ?= install endif -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -PREFIX ?= /usr -else PREFIX ?= /usr/local -endif - DESTDIR ?= BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else MANDIR ?= $(PREFIX)/share/man/man1 +endif INSTALL_PROGRAM ?= $(INSTALL) -m 755 INSTALL_SCRIPT ?= $(INSTALL) -m 755 -- cgit v0.12 From c8b31263ccbf687c4ea05f29da2011462d1e8e5b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 13:08:38 +0100 Subject: added FUZ_CLEVEL_DEFAULT --- lib/lz4hc.c | 2 +- tests/fuzzer.c | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index ae245b5..b2930de 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -564,7 +564,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; return dictSize; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index ff02e0c..dbc15ec 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -63,6 +63,7 @@ typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which do #define FUZ_MAX_BLOCK_SIZE (1 << 17) #define FUZ_MAX_DICT_SIZE (1 << 15) #define FUZ_COMPRESSIBILITY_DEFAULT 60 +#define FUZ_CLEVEL_DEFAULT LZ4HC_CLEVEL_OPT_MIN #define PRIME1 2654435761U #define PRIME2 2246822519U #define PRIME3 3266489917U @@ -349,18 +350,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9); + ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); /* Test compression */ @@ -466,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -491,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, FUZ_CLEVEL_DEFAULT); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -592,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); + LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1)); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); -- cgit v0.12 From 12aae846b3006d7cd76a1197992eab1a0e995466 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 13:19:11 +0100 Subject: fixed table update in LZ4_loadDictHC --- lib/lz4hc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b2930de..3097df2 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -564,8 +564,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) + LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); + else + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; } -- cgit v0.12 From 98f9d6c726605e8138ef3400f46fc845e30c995f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 14:04:38 +0100 Subject: improved logging --- lib/lz4opt.h | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 46797ef..1633453 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -33,6 +33,7 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ +//#include #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) @@ -205,7 +206,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -222,7 +223,7 @@ static int LZ4HC_compress_optimal ( { LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; - const BYTE *inr; + const BYTE *inr = NULL; size_t res, cur, cur2; size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; @@ -245,7 +246,7 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), (int)match_num, (int)last_pos); if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -259,7 +260,7 @@ static int LZ4HC_compress_optimal ( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; - LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); @@ -279,28 +280,28 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur-1].litlen + 1; if (cur != litlen) { price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)(cur - litlen), opt[cur - litlen].price, (int)price, (int)cur, (int)litlen); } else { price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)llen); } } else { litlen = 1; price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen)); + LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)LZ4HC_literalsPrice(litlen)); } mlen = 1; best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, opt[cur].price); if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-(const BYTE*)source), (int)match_num); if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -314,7 +315,7 @@ static int LZ4HC_compress_optimal ( mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; - LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)cur2, matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { @@ -329,7 +330,7 @@ static int LZ4HC_compress_optimal ( price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-(const BYTE*)source), (int)mlen, (int)best_mlen, matches[i].off, (int)price, (int)litlen, (int)(cur - litlen), opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } @@ -344,10 +345,10 @@ static int LZ4HC_compress_optimal ( encode: /* cur, last_pos, best_mlen, best_off have to be set */ for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); } - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off); + LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off); opt[0].mlen = 1; while (1) { @@ -362,13 +363,13 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ } for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); + LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); i += opt[i].mlen; } cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); + LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; @@ -380,7 +381,7 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ if (res) return 0; - LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset); + LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-(const BYTE*)source), (int)offset); } } -- cgit v0.12 From 1c80b9af4edf94f93582d40bae0856d13eacbf11 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 15:18:19 +0100 Subject: LZ4HC_getSearchNum --- lib/lz4hc.c | 16 ++++++++++++++-- lib/lz4opt.h | 18 +++++++++++------- tests/fuzzer.c | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 3097df2..8ca5442 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -486,6 +486,14 @@ _Search3: return (int) (((char*)op)-dest); } +static int LZ4HC_getSearchNum(int compressionLevel) +{ + switch (compressionLevel) { + default: return 0; /* unused */ + case 11: return 128; + case 12: return 1<<10; + } +} static int LZ4HC_compress_generic ( LZ4HC_CCtx_internal* const ctx, @@ -497,13 +505,14 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { + // printf("LZ4HC_compress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); - case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); + case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); default: - case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); + case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); } } return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); @@ -554,11 +563,13 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; + LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + // printf("LZ4_loadDictHC dictSize=%d\n", (int)dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -577,6 +588,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { + // printf("LZ4HC_setExternalDict\n"); if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 1633453..041df52 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -37,6 +37,7 @@ #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ4_LOG_TREE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_OPT_NUM (1<<12) @@ -106,6 +107,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; + LZ4_LOG_TREE("LZ4HC_BinTree_InsertAndGetAllMatches nbAttempts=%d\n", nbAttempts); + if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -117,6 +120,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); + LZ4_LOG_TREE("matchIndex[%u] current[%u] lowLimit[%u]\n", matchIndex, current, lowLimit); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { @@ -141,29 +145,30 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matches[mnum].len = (int)matchLength; mnum++; } - if (best_mlen > LZ4_OPT_NUM) break; + if (best_mlen > LZ4_OPT_NUM) { LZ4_LOG_TREE("best_mlen > LZ4_OPT_NUM\n"); break; } } if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + { LZ4_LOG_TREE("ip+matchLength > iHighLimit\n"); break; } /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) break; + if (*ptr0 == (U16)-1) { LZ4_LOG_TREE("*ptr0 == (U16)-1"); break; } delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) break; + if (*ptr1 == (U16)-1) { LZ4_LOG_TREE("*ptr1 == (U16)-1\n"); break; } delta1 = *ptr1; delta0 += delta1; matchIndex -= delta1; } } + LZ4_LOG_TREE("nbAttempts=%d mnum=%d\n", nbAttempts, mnum); *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; @@ -178,7 +183,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdateBT; - + LZ4_LOG_TREE("LZ4HC_updateBinTree %d->%d nbAttempts=%d\n", idx, target, ctx->searchNum); while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -206,7 +211,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + LZ4_LOG_PARSER("%u: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -330,7 +335,6 @@ static int LZ4HC_compress_optimal ( price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); } - LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-(const BYTE*)source), (int)mlen, (int)best_mlen, matches[i].off, (int)price, (int)litlen, (int)(cur - litlen), opt[cur - litlen].price); if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index dbc15ec..fb918b4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -601,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); -- cgit v0.12 From 3d5bb38977cbb200fe0950095b7a10261b7ff792 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 15:37:12 +0100 Subject: clean logging --- lib/lz4hc.c | 3 --- lib/lz4opt.h | 49 ++++--------------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8ca5442..45dea72 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -505,7 +505,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - // printf("LZ4HC_compress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel > 9) { switch (compressionLevel) { @@ -569,7 +568,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - // printf("LZ4_loadDictHC dictSize=%d\n", (int)dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -588,7 +586,6 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { - // printf("LZ4HC_setExternalDict\n"); if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); else diff --git a/lib/lz4opt.h b/lib/lz4opt.h index 041df52..cc80495 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -33,13 +33,6 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -//#include -#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LZ4_LOG_TREE(fmt, ...) //printf(fmt, __VA_ARGS__) - - #define LZ4_OPT_NUM (1<<12) @@ -107,8 +100,6 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( size_t matchLength = 0; U32* HashPos; - LZ4_LOG_TREE("LZ4HC_BinTree_InsertAndGetAllMatches nbAttempts=%d\n", nbAttempts); - if (ip + MINMATCH > iHighLimit) return 1; /* HC4 match finder */ @@ -120,7 +111,6 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( ptr1 = &DELTANEXTMAXD(current*2); delta0 = delta1 = (U16)(current - matchIndex); - LZ4_LOG_TREE("matchIndex[%u] current[%u] lowLimit[%u]\n", matchIndex, current, lowLimit); while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { @@ -145,30 +135,29 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( matches[mnum].len = (int)matchLength; mnum++; } - if (best_mlen > LZ4_OPT_NUM) { LZ4_LOG_TREE("best_mlen > LZ4_OPT_NUM\n"); break; } + if (best_mlen > LZ4_OPT_NUM) break; } if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ - { LZ4_LOG_TREE("ip+matchLength > iHighLimit\n"); break; } /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ if (*(ip+matchLength) < *(match+matchLength)) { *ptr0 = delta0; ptr0 = &DELTANEXTMAXD(matchIndex*2); - if (*ptr0 == (U16)-1) { LZ4_LOG_TREE("*ptr0 == (U16)-1"); break; } + if (*ptr0 == (U16)-1) break; delta0 = *ptr0; delta1 += delta0; matchIndex -= delta0; } else { *ptr1 = delta1; ptr1 = &DELTANEXTMAXD(matchIndex*2+1); - if (*ptr1 == (U16)-1) { LZ4_LOG_TREE("*ptr1 == (U16)-1\n"); break; } + if (*ptr1 == (U16)-1) break; delta1 = *ptr1; delta0 += delta1; matchIndex -= delta1; } } - LZ4_LOG_TREE("nbAttempts=%d mnum=%d\n", nbAttempts, mnum); *ptr0 = (U16)-1; *ptr1 = (U16)-1; if (matchNum) *matchNum = mnum; @@ -183,7 +172,6 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); U32 idx = ctx->nextToUpdateBT; - LZ4_LOG_TREE("LZ4HC_updateBinTree %d->%d nbAttempts=%d\n", idx, target, ctx->searchNum); while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -211,7 +199,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( opt[pos].off = (int)offset; \ opt[pos].litlen = (int)litlen; \ opt[pos].price = (int)price; \ - LZ4_LOG_PARSER("%u: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ } @@ -251,7 +238,6 @@ static int LZ4HC_compress_optimal ( llen = ip - anchor; match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); if (!match_num) { ip++; continue; } - LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), (int)match_num, (int)last_pos); if ((size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; @@ -265,7 +251,6 @@ static int LZ4HC_compress_optimal ( for (i = 0; i < match_num; i++) { mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; - LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-(const BYTE*)source), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { litlen = 0; price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); @@ -285,29 +270,22 @@ static int LZ4HC_compress_optimal ( litlen = opt[cur-1].litlen + 1; if (cur != litlen) { price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)(cur - litlen), opt[cur - litlen].price, (int)price, (int)cur, (int)litlen); } else { price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); - LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)llen); } } else { litlen = 1; price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); - LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, (int)litlen, (int)LZ4HC_literalsPrice(litlen)); } mlen = 1; best_mlen = 0; - LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-(const BYTE*)source), (int)price, (int)cur, opt[cur].price); if (cur > last_pos || price < (size_t)opt[cur].price) SET_PRICE(cur, mlen, best_mlen, litlen, price); if (cur == last_pos || inr >= mflimit) break; - LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); - LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-(const BYTE*)source), (int)match_num); - if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -320,7 +298,6 @@ static int LZ4HC_compress_optimal ( mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; cur2 = cur; best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; - LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-(const BYTE*)source), (int)cur, (int)cur2, matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); while (mlen <= best_mlen) { if (opt[cur2].mlen == 1) { @@ -348,12 +325,6 @@ static int LZ4HC_compress_optimal ( cur = last_pos - best_mlen; encode: /* cur, last_pos, best_mlen, best_off have to be set */ - for (i = 1; i <= last_pos; i++) { - LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); - } - - LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off); - opt[0].mlen = 1; while (1) { mlen = opt[cur].mlen; @@ -366,26 +337,15 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ cur -= mlen; } - for (i = 0; i <= last_pos;) { - LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+i), (int)i, (int)last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen); - i += opt[i].mlen; - } - cur = 0; while (cur < last_pos) { - LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-(const BYTE*)source+cur), (int)cur, (int)last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen); res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); - LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); - if (res) return 0; - - LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-(const BYTE*)source), (int)offset); } } @@ -394,7 +354,6 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun< Date: Wed, 28 Dec 2016 15:38:59 +0100 Subject: removed nextToUpdateBT --- lib/lz4hc.c | 5 ++--- lib/lz4hc.h | 2 -- lib/lz4opt.h | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 45dea72..5d4ea3e 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB; + hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; @@ -597,7 +597,7 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */ + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, @@ -657,7 +657,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; - if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit; } return dictSize; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index ad731d9..1036fd0 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -154,7 +154,6 @@ typedef struct uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ - uint32_t nextToUpdateBT; /* index from which to continue binary tree update */ uint32_t searchNum; /* only for optimal parser */ uint32_t compressionLevel; } LZ4HC_CCtx_internal; @@ -172,7 +171,6 @@ typedef struct unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ - unsigned int nextToUpdateBT; /* index from which to continue binary tree update */ unsigned int searchNum; /* only for optimal parser */ unsigned int compressionLevel; } LZ4HC_CCtx_internal; diff --git a/lib/lz4opt.h b/lib/lz4opt.h index cc80495..d1913fe 100644 --- a/lib/lz4opt.h +++ b/lib/lz4opt.h @@ -171,7 +171,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons { const BYTE* const base = ctx->base; const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdateBT; + U32 idx = ctx->nextToUpdate; while(idx < target) idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); } @@ -184,10 +184,10 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) { int mnum = 0; - if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */ + if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); - ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen); + ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); return mnum; } -- cgit v0.12 From d2b51c22d300ff67ef3cd91b905a4c35ff876515 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 28 Dec 2016 17:47:10 +0100 Subject: fuzzer: tests more compression levels --- tests/fuzzer.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fb918b4..b129c96 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -63,7 +63,6 @@ typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which do #define FUZ_MAX_BLOCK_SIZE (1 << 17) #define FUZ_MAX_DICT_SIZE (1 << 15) #define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define FUZ_CLEVEL_DEFAULT LZ4HC_CLEVEL_OPT_MIN #define PRIME1 2654435761U #define PRIME2 2246822519U #define PRIME3 3266489917U @@ -304,6 +303,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart); + int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); char* const block = ((char*)CNBuffer) + blockStart; const char* dict = block - dictSize; int compressedSize, HCcompressedSize; @@ -350,13 +350,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; /* Test compression HC using external state */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); /* Test compression using external state */ @@ -467,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); /* Test HC compression with output size being exactly what's necessary (should work) */ FUZ_DISPLAYTEST; - ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); /* Test compression with missing bytes into output buffer => must fail */ @@ -492,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, FUZ_CLEVEL_DEFAULT); + ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes); FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes) } @@ -593,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST; dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1)); + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed"); @@ -656,7 +656,7 @@ _output_error: #define testCompressedSize (128 KB) #define ringBufferSize (8 KB) -static void FUZ_unitTests(void) +static void FUZ_unitTests(int compressionLevel) { const unsigned testNb = 0; const unsigned seed = 0; @@ -751,7 +751,7 @@ static void FUZ_unitTests(void) /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); @@ -762,7 +762,7 @@ static void FUZ_unitTests(void) /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); @@ -776,7 +776,7 @@ static void FUZ_unitTests(void) { int result1, result2; int segSize = testCompressedSize / 2; crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1); @@ -793,7 +793,7 @@ static void FUZ_unitTests(void) /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 32 KB); result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); @@ -814,7 +814,7 @@ static void FUZ_unitTests(void) int segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, dict, dictSize); XXH64_reset(&crcOrigState, 0); @@ -860,7 +860,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); while (iNext + messageSize < testCompressedSize) { @@ -902,7 +902,7 @@ static void FUZ_unitTests(void) XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); - LZ4_resetStreamHC(&sHC, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); LZ4_setStreamDecode(&decodeState, NULL, 0); #define BSIZE1 65537 @@ -957,7 +957,7 @@ static void FUZ_unitTests(void) } } - printf("All unit tests completed successfully \n"); + printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel); return; _output_error: exit(1); @@ -1103,7 +1103,7 @@ int main(int argc, const char** argv) if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - if ((seedset==0) && (testNb==0)) FUZ_unitTests(); + if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } if (nbTests<=0) nbTests=1; -- cgit v0.12 From 9e867db90adeacd3f2e241c1f3d1097640c69c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 10:14:35 +0100 Subject: cmake: Fix SOVERSION to match Makefiles Fix SOVERSION to use only major lz4 version, as Makefiles do. This ensure that CMake uses 'liblz.so.1' SONAME and creates 'liblz.so.1' symlink. --- contrib/cmake_unofficial/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index de070d6..2380a0b 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -99,7 +99,7 @@ endif() # liblz4 add_library(lz4 ${LZ4_SOURCES}) set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_STRING}" + SOVERSION "${LZ4_VERSION_MAJOR}" VERSION "${LZ4_VERSION_STRING}" POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) -- cgit v0.12 From 28db4acc90805c8bcc183b788900ac566b0346d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 10:34:12 +0100 Subject: cmake: Support building shared & static libs simultaneously Add an additional BUILD_STATIC_LIBS option to control building static libraries independently of shared. This makes it possible (if both options are set to ON) to build both shared and static libraries simulataneously. A dependant option is used to preserve the current BUILD_SHARED_LIBS behavior, i.e. -DBUILD_SHARED_LIBS=ON -- shared lib only, -DBUILD_SHARED_LIBS=OFF -- static lib only. The targets used to build shared and static library are split now, and only relevant properties are passed to each of them. An alias is used to link programs to the preferred library. --- contrib/cmake_unofficial/CMakeLists.txt | 44 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index 2380a0b..df16d46 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -65,6 +65,11 @@ endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED) # which case we always use static libraries. include(CMakeDependentOption) CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF) +CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON) + +if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS) + message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled") +endif() set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib") set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs") @@ -90,28 +95,41 @@ set(LZ4_CLI_SOURCES # we're building a shared library this is ignored and PIC is always # used. option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON) -if(LZ4_POSITION_INDEPENDENT_LIB OR BUILD_SHARED_LIBS) - set(LZ4_POSITION_INDEPENDENT_CODE TRUE) -else() - set(LZ4_POSITION_INDEPENDENT_CODE FALSE) -endif() # liblz4 -add_library(lz4 ${LZ4_SOURCES}) -set_target_properties(lz4 PROPERTIES - SOVERSION "${LZ4_VERSION_MAJOR}" - VERSION "${LZ4_VERSION_STRING}" - POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_CODE}) +set(LZ4_LIBRARIES_BUILT) +if(BUILD_SHARED_LIBS) + add_library(lz4_shared SHARED ${LZ4_SOURCES}) + set_target_properties(lz4_shared PROPERTIES + OUTPUT_NAME lz4 + SOVERSION "${LZ4_VERSION_MAJOR}" + VERSION "${LZ4_VERSION_STRING}") + list(APPEND LZ4_LIBRARIES_BUILT lz4_shared) +endif() +if(BUILD_STATIC_LIBS) + add_library(lz4_static STATIC ${LZ4_SOURCES}) + set_target_properties(lz4_static PROPERTIES + OUTPUT_NAME lz4 + POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB}) + list(APPEND LZ4_LIBRARIES_BUILT lz4_static) +endif() + +# link to shared whenever possible, to static otherwise +if(BUILD_SHARED_LIBS) + set(LZ4_LINK_LIBRARY lz4_shared) +else() + set(LZ4_LINK_LIBRARY lz4_static) +endif() # lz4 add_executable(lz4cli ${LZ4_CLI_SOURCES}) set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4) -target_link_libraries(lz4cli lz4) +target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY}) # lz4c add_executable(lz4c ${LZ4_CLI_SOURCES}) set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS") -target_link_libraries(lz4c lz4) +target_link_libraries(lz4c ${LZ4_LINK_LIBRARY}) # Extra warning flags include (CheckCCompilerFlag) @@ -149,7 +167,7 @@ if(NOT LZ4_BUNDLED_MODE) install(TARGETS lz4cli lz4c RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") - install(TARGETS lz4 + install(TARGETS ${LZ4_LIBRARIES_BUILT} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") install(FILES -- cgit v0.12 From 1380c33b7349507e1d51b505a2f4ee8ab126c587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 15:33:15 +0100 Subject: cmake: Install lz4cat and unlz4 symlinks --- contrib/cmake_unofficial/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index df16d46..38f3b00 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -177,6 +177,18 @@ if(NOT LZ4_BUNDLED_MODE) DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") + + # install lz4cat and unlz4 symlinks on *nix + if(UNIX) + install(CODE " + foreach(f lz4cat unlz4) + set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\") + message(STATUS \"Symlinking: \${dest} -> lz4\") + execute_process( + COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") + endforeach() + ") + endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) # pkg-config -- cgit v0.12 From d7969e49af22943a3f0062b8fe8abc04bb1dd6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 2 Jan 2017 15:57:49 +0100 Subject: cmake: Install manpages --- contrib/cmake_unofficial/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index 38f3b00..9a0983d 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -175,6 +175,8 @@ if(NOT LZ4_BUNDLED_MODE) "${LZ4_LIB_SOURCE_DIR}/lz4frame.h" "${LZ4_LIB_SOURCE_DIR}/lz4hc.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") @@ -188,6 +190,13 @@ if(NOT LZ4_BUNDLED_MODE) COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\") endforeach() ") + + # create manpage aliases + foreach(f lz4cat unlz4) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") + endforeach() endif(UNIX) endif(NOT LZ4_BUNDLED_MODE) -- cgit v0.12 From 05e27ade68e8b0db8f0965d218199b024e6fb217 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 3 Jan 2017 03:59:27 +0100 Subject: updated NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index d149128..1d42954 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,11 @@ v1.7.5 -lz4hc : new high compression mode : levels 10-12 compress more and slower, by @inikep. +lz4hc : new high compression mode : levels 10-12 compress more and slower, by Przemyslaw Skibinski lz4cat : fix : works with relative path (#284) and stdin (#285) (reported by @beiDei8z) cli : fix minor notification when using -r recursive mode API : lz4frame : LZ4F_frameBound(0) gives upper bound of *flush() and *End() operations (#290, #280) doc : markdown version of man page, by Takayuki Matsuoka (#279) build : Makefile : fix make -jX lib+exe concurrency (#277) +build : cmake : improvements by Michał Górny (#296) v1.7.4.2 fix : Makefile : release build compatible with PIE and customized compilation directives provided through environment variables (#274, reported by Antoine Martin) -- cgit v0.12