diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2017-01-03 05:09:52 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-01-03 05:09:52 (GMT) |
commit | 7bb64ff2b69a9f8367de9ab483cdadf42b4c1b65 (patch) | |
tree | 35151b46f6ff17ff4fa8c8cfd53ab6e614c8fc97 /programs | |
parent | 018ddf799917ee5c68b5266d6f42277fa6750080 (diff) | |
parent | 16d69c8d039e42defbf7387dd0e4e9318017eb6d (diff) | |
download | lz4-7bb64ff2b69a9f8367de9ab483cdadf42b4c1b65.zip lz4-7bb64ff2b69a9f8367de9ab483cdadf42b4c1b65.tar.gz lz4-7bb64ff2b69a9f8367de9ab483cdadf42b4c1b65.tar.bz2 |
Merge pull request #297 from lz4/devv1.7.5
v1.7.5
Diffstat (limited to 'programs')
-rw-r--r-- | programs/.gitignore | 2 | ||||
-rw-r--r-- | programs/Makefile | 118 | ||||
-rw-r--r-- | programs/bench.c | 19 | ||||
-rw-r--r-- | programs/datagen.c | 33 | ||||
-rw-r--r-- | programs/lz4.1 | 411 | ||||
-rw-r--r-- | programs/lz4.1.md | 219 | ||||
-rw-r--r-- | programs/lz4cli.c | 104 | ||||
-rw-r--r-- | programs/lz4io.c | 67 | ||||
-rw-r--r-- | programs/platform.h | 143 | ||||
-rw-r--r-- | programs/util.h | 152 |
10 files changed, 797 insertions, 471 deletions
diff --git a/programs/.gitignore b/programs/.gitignore index df3423e..daa7f14 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -1,5 +1,7 @@ # local binary (Makefile) lz4 +unlz4 +lz4cat lz4c lz4c32 datagen diff --git a/programs/Makefile b/programs/Makefile index 504416f..060ce21 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,12 +27,21 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # ########################################################################## -DESTDIR ?= -PREFIX ?= /usr/local -BINDIR := $(PREFIX)/bin -MANDIR := $(PREFIX)/share/man/man1 +# Version numbers +LIBVER_SRC := ../lib/lz4.h +LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) +LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) +LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) + LZ4DIR := ../lib -VOID := /dev/null + +SRCFILES := $(wildcard $(LZ4DIR)/*.c) $(wildcard *.c) +OBJFILES := $(patsubst %.c,%.o,$(SRCFILES)) CPPFLAGS+= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ CFLAGS ?= -O3 @@ -42,68 +51,115 @@ DEBUGFLAGS:=-g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(MOREFLAGS) FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) +LZ4_VERSION=$(LIBVER) +MD2ROFF =ronn +MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="lz4 $(LZ4_VERSION)" + # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe +VOID := nul +EXT :=.exe else -EXT = +VOID := /dev/null +EXT := endif default: lz4-release -lz4-release: $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.o datagen.o - $(CC) $(FLAGS) $^ -o lz4$(EXT) - all: lz4 lz4c all32: CFLAGS+=-m32 all32: all lz4: CFLAGS += $(DEBUGFLAGS) -lz4: lz4-release +lz4: $(OBJFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lz4-release: DEBUGFLAGS= +lz4-release: lz4 -lz4c: CFLAGS += $(DEBUGFLAGS) -lz4c : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o bench.o lz4io.o lz4cli.c datagen.o +lz4c : CFLAGS += $(DEBUGFLAGS) +lz4c : $(SRCFILES) $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c32: CFLAGS+=-m32 -lz4c32: lz4 - @cp lz4$(EXT) lz4c32$(EXT) +lz4c32: CFLAGS += -m32 $(DEBUGFLAGS) +lz4c32: $(SRCFILES) + $(CC) $(FLAGS) $^ -o $@$(EXT) clean: @$(MAKE) -C $(LZ4DIR) $@ > $(VOID) @$(RM) core *.o *.test tmp* \ - lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) unlz4 lz4cat @echo Cleaning completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and -#FreeBSD targets -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD)) +lz4.1: lz4.1.md + cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ + +man: lz4.1 + +clean-man: + rm lz4.1 + +preview-man: clean-man man + man ./lz4.1 + + +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +unlz4: lz4 + ln -s lz4 unlz4 + +lz4cat: lz4 + ln -s lz4 lz4cat + + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +PREFIX ?= /usr/local +DESTDIR ?= +BINDIR ?= $(PREFIX)/bin + +ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) +MANDIR ?= $(PREFIX)/man/man1 +else +MANDIR ?= $(PREFIX)/share/man/man1 +endif + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 +INSTALL_SCRIPT ?= $(INSTALL) -m 755 +INSTALL_MAN ?= $(INSTALL) -m 644 + install: lz4$(EXT) lz4c$(EXT) @echo Installing binaries - @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @$(INSTALL_PROGRAM) lz4 $(DESTDIR)$(BINDIR)/lz4 + @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat + @ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4 + @$(INSTALL_PROGRAM) lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c @echo Installing man pages - @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @$(INSTALL_MAN) -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1 @ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1 @echo lz4 installation completed uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/lz4cat$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/unlz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4$(EXT) - @$(RM) $(DESTDIR)$(BINDIR)/lz4c$(EXT) + @$(RM) $(DESTDIR)$(BINDIR)/lz4cat + @$(RM) $(DESTDIR)$(BINDIR)/unlz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4 + @$(RM) $(DESTDIR)$(BINDIR)/lz4c @$(RM) $(DESTDIR)$(MANDIR)/lz4.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4c.1 @$(RM) $(DESTDIR)$(MANDIR)/lz4cat.1 diff --git a/programs/bench.c b/programs/bench.c index 434da8b..8311503 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -24,13 +24,22 @@ */ +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* memset */ -#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <stdio.h> /* fprintf, fopen, ftello */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include "datagen.h" /* RDG_genBuffer */ @@ -170,7 +179,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init */ - if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1; + if (cLevel < LZ4HC_CLEVEL_MIN) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -494,8 +503,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, { double const compressibility = (double)g_compressibilityDefault / 100; - if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL; - if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL; + if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; + if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); diff --git a/programs/datagen.c b/programs/datagen.c index f97c2d0..a61afc0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -26,43 +26,14 @@ /************************************** * Includes **************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ #include <stdlib.h> /* malloc */ #include <stdio.h> /* FILE, fwrite */ #include <string.h> /* memcpy */ /************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -#else -# define SET_BINARY_MODE(file) -#endif - - -/************************************** * Constants **************************************/ #define KB *(1 <<10) diff --git a/programs/lz4.1 b/programs/lz4.1 index 2c94033..48c988a 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,251 +1,206 @@ -\." -\." lz4.1: This is a manual page for 'lz4' program. This file is part of the -\." lz4 <http://www.lz4.org/> project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lz4 "1" "2015-03-21" "lz4" "User Commands" -.SH NAME -\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> -.PP -.B unlz4 -is equivalent to -.BR "lz4 \-d" -.br -.B lz4cat -is equivalent to -.BR "lz4 \-dcfm" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lz4 -with appropriate arguments -.RB ( "lz4 \-d" -or -.BR "lz4 \-dc" ) -instead of the names -.B unlz4 -and -.BR lz4cat . - - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .lz4 -format. - -.B lz4 -supports a command line syntax similar \fIbut not identical\fR to -.BR gzip (1). -Differences are : - \fBlz4\fR preserves original files - \fBlz4\fR compresses a single file by default (use \fB-m\fR for multiple files) - \fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 - When no destination name is provided, compressed file name receives a \fB.lz4\fR suffix - When no destination name is provided, if \fBstdout\fR is \fInot\fR the console, it becomes the output (like a silent \fB-c\fR) - Therefore \fBlz4 file > /dev/null\fR will not create \fBfile.lz4\fR - \fBlz4 file\fR shows real-time statistics during compression (use \fB-q\fR to silent them) - -Default behaviors can be modified by opt-in commands, described below. -\fBlz4 --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .lz4 files" -It is possible to concatenate -.B .lz4 -files as is. -.B lz4 -will decompress such files as if they were a single -.B .lz4 -file. For example: - lz4 file1 > foo.lz4 - lz4 file2 >> foo.lz4 - then - lz4cat foo.lz4 - is equivalent to : - cat file1 file2 - -.PP - -.SH OPTIONS +. +.TH "LZ4" "1" "November 2016" "lz4 1.7.4" "User Commands" +. +.SH "NAME" +\fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files +. +.SH "SYNOPSIS" +\fBlz4\fR [\fIOPTIONS\fR] [\-|INPUT\-FILE] \fIOUTPUT\-FILE\fR +. +.P +\fBunlz4\fR is equivalent to \fBlz4 \-d\fR +. +.P +\fBlz4cat\fR is equivalent to \fBlz4 \-dcfm\fR +. +.P +When writing scripts that need to decompress files, it is recommended to always use the name \fBlz4\fR with appropriate arguments (\fBlz4 \-d\fR or \fBlz4 \-dc\fR) instead of the names \fBunlz4\fR and \fBlz4cat\fR\. +. +.SH "DESCRIPTION" +\fBlz4\fR is an extremely fast lossless compression algorithm, based on \fBbyte\-aligned LZ77\fR family of compression scheme\. \fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi\-core CPUs\. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limit on multi\-core systems\. The native file format is the \fB\.lz4\fR format\. +. +.SS "Difference between lz4 and gzip" +\fBlz4\fR supports a command line syntax similar \fIbut not identical\fR to \fBgzip(1)\fR\. Differences are : +. +.IP "\(bu" 4 +\fBlz4\fR preserves original files +. +.IP "\(bu" 4 +\fBlz4\fR compresses a single file by default (see \fB\-m\fR for multiple files) +. +.IP "\(bu" 4 +\fBlz4 file1 file2\fR means : compress file1 \fIinto\fR file2 +. +.IP "\(bu" 4 +\fBlz4\fR shows real\-time notification statistics during compression or decompression of a single file (use \fB\-q\fR to silent them) +. +.IP "\(bu" 4 +If no destination name is provided, result is sent to \fBstdout\fR \fIexcept if stdout is the console\fR\. +. +.IP "\(bu" 4 +If no destination name is provided, \fBand\fR if \fBstdout\fR is the console, \fBfile\fR is compressed into \fBfile\.lz4\fR\. +. +.IP "\(bu" 4 +As a consequence of previous rules, note the following example : \fBlz4 file | consumer\fR sends compressed data to \fBconsumer\fR through \fBstdout\fR, hence it does \fInot\fR create any \fBfile\.lz4\fR\. +. +.IP "" 0 +. +.P +Default behaviors can be modified by opt\-in commands, detailed below\. +. +.IP "\(bu" 4 +\fBlz4 \-m\fR makes it possible to provide multiple input filenames, which will be compressed into files using suffix \fB\.lz4\fR\. Progress notifications are also disabled by default\. This mode has a behavior which more closely mimics \fBgzip\fR command line, with the main difference being that source files are preserved by default\. +. +.IP "\(bu" 4 +It\'s possible to opt\-in to erase source files on successful compression or decompression, using \fB\-\-rm\fR command\. +. +.IP "\(bu" 4 +Consequently, \fBlz4 \-m \-\-rm\fR behaves the same as \fBgzip\fR\. +. +.IP "" 0 +. +.SS "Concatenation of \.lz4 files" +It is possible to concatenate \fB\.lz4\fR files as is\. \fBlz4\fR will decompress such files as if they were a single \fB\.lz4\fR file\. For example: lz4 file1 > foo\.lz4 lz4 file2 >> foo\.lz4 +. +.P +then lz4cat foo\.lz4 +. +.P +is equivalent to : cat file1 file2 +. +.SH "OPTIONS" . .SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. +In some cases, some options can be expressed using short command \fB\-x\fR or long command \fB\-\-long\-word\fR\. Short commands can be concatenated together\. For example, \fB\-d \-c\fR is equivalent to \fB\-dc\fR\. Long commands cannot be concatenated\. They must be clearly separated by a space\. +. .SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. +When multiple contradictory commands are issued on a same command line, only the latest one will be applied\. . .SS "Operation mode" +. .TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlz4 -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .lz4 -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .lz4 -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .lz4 -extension. -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .lz4 -files. -The decompressed data is discarded. -No files are created nor removed. - -.TP -.BR \-b# -Benchmark mode, using # compression level. - +\fB\-z\fR \fB\-\-compress\fR +Compress\. This is the default operation mode when no operation mode option is specified, no other operation mode is implied from the command name (for example, \fBunlz4\fR implies \fB\-\-decompress\fR), nor from the input file name (for example, a file extension \fB\.lz4\fR implies \fB\-\-decompress\fR by default)\. \fB\-z\fR can also be used to force compression of an already compressed \fB\.lz4\fR file\. +. +.TP +\fB\-d\fR \fB\-\-decompress\fR \fB\-\-uncompress\fR +Decompress\. \fB\-\-decompress\fR is also the default operation when the input filename has an \fB\.lz4\fR extension\. +. +.TP +\fB\-t\fR \fB\-\-test\fR +Test the integrity of compressed \fB\.lz4\fR files\. The decompressed data is discarded\. No files are created nor removed\. +. +.TP +\fB\-b#\fR +Benchmark mode, using \fB#\fR compression level\. . .SS "Operation modifiers" +. +.TP +\fB\-#\fR +Compression level, with # being any value from 1 to 16\. Higher values trade compression speed for compression ratio\. Values above 16 are considered the same as 16\. Recommended values are 1 for fast compression (default), and 9 for high compression\. Speed/compression trade\-off will vary depending on data to compress\. Decompression speed remains fast at all settings\. +. .TP -.B \-# - compression level, with # being any value from 1 to 16. - Higher values trade compression speed for compression ratio. - Values above 16 are considered the same as 16. - Recommended values are 1 for fast compression (default), and 9 for high compression. - Speed/compression trade-off will vary depending on data to compress. - Decompression speed remains fast at all settings. - -.TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lz4 -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lz4cat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lz4 . -.RE - -.TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - -.TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, it is possible to specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.lz4 -. - -.TP -.B \-B# - block size [4-7](default : 7) - B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB -.TP -.B \-BD - block dependency (improves compression ratio on small blocks) -.TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) -.TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. -.TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) -.TP -.B \-l - use Legacy format (typically used for Linux Kernel compression) - note : \fB-l\fR is not compatible with \fB-m\fR (\fB--multiple\fR) +\fB\-f\fR \fB\-\-[no\-]force\fR +This option has several effects: +. +.IP +If the target file already exists, overwrite it without prompting\. +. +.IP +When used with \fB\-\-decompress\fR and \fBlz4\fR cannot recognize the type of the source file, copy the source file as is to standard output\. This allows \fBlz4cat \-\-force\fR to be used like \fBcat (1)\fR for files that have not been compressed with \fBlz4\fR\. +. +.TP +\fB\-c\fR \fB\-\-stdout\fR \fB\-\-to\-stdout\fR +Force write to standard output, even if it is the console\. +. +.TP +\fB\-m\fR \fB\-\-multiple\fR +Multiple input files\. Compressed file names will be appended a \fB\.lz4\fR suffix\. This mode also reduces notification level\. \fBlz4 \-m\fR has a behavior equivalent to \fBgzip \-k\fR (it preserves source files by default)\. +. +.TP +\fB\-r\fR +operate recursively on directories\. This mode also sets \fB\-m\fR (multiple input files)\. +. +.TP +\fB\-B#\fR +Block size [4\-7](default : 7) +. +.br +\fB\-B4\fR= 64KB ; \fB\-B5\fR= 256KB ; \fB\-B6\fR= 1MB ; \fB\-B7\fR= 4MB +. +.TP +\fB\-BD\fR +Block Dependency (improves compression ratio on small blocks) +. +.TP +\fB\-\-[no\-]frame\-crc\fR +Select frame checksum (default:enabled) +. +.TP +\fB\-\-[no\-]content\-size\fR +Header includes original size (default:not present) +. +.br +Note : this option can only be activated when the original size can be determined, hence for a file\. It won\'t work with unknown source size, such as stdin or pipe\. +. +.TP +\fB\-\-[no\-]sparse\fR +Sparse mode support (default:enabled on file, disabled on stdout) +. +.TP +\fB\-l\fR +Use Legacy format (typically for Linux Kernel compression) +. +.br +Note : \fB\-l\fR is not compatible with \fB\-m\fR (\fB\-\-multiple\fR) nor \fB\-r\fR . .SS "Other options" +. .TP -.BR \-v ", " --verbose - verbose mode +\fB\-v\fR \fB\-\-verbose\fR +Verbose mode +. .TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too +\fB\-q\fR \fB\-\-quiet\fR +Suppress warnings and real\-time statistics; specify twice to suppress errors too +. .TP -.B \-h/\-H ", " --help - display help/long help and exit +\fB\-h\fR \fB\-H\fR \fB\-\-help\fR +Display help/long help and exit +. .TP -.BR \-V ", " \--version - display Version number and exit +\fB\-V\fR \fB\-\-version\fR +Display Version number and exit +. +.TP +\fB\-k\fR \fB\-\-keep\fR +Preserve source files (default behavior) +. .TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. - +\fB\-\-rm\fR +Delete source files on successful compression or decompression . .SS "Benchmark mode" +. .TP -.B \-b# - benchmark file(s), using # compression level +\fB\-b#\fR +Benchmark file(s), using # compression level +. .TP -.B \-e# - benchmark multiple compression levels, from b# to e# (included) +\fB\-e#\fR +Benchmark multiple compression levels, from b# to e# (included) +. .TP -.B \-i# - minimum evaluation in seconds [1-9] (default : 3) +\fB\-i#\fR +Minimum evaluation in seconds [1\-9] (default : 3) +. .TP -.B \-r - operate recursively on directories - - -.SH BUGS -Report bugs at: https://github.com/Cyan4973/lz4/issues - -.SH AUTHOR +\fB\-r\fR +Operate recursively on directories +. +.SH "BUGS" +Report bugs at: https://github\.com/lz4/lz4/issues +. +.SH "AUTHOR" Yann Collet diff --git a/programs/lz4.1.md b/programs/lz4.1.md new file mode 100644 index 0000000..48f3152 --- /dev/null +++ b/programs/lz4.1.md @@ -0,0 +1,219 @@ +lz4(1) -- lz4, unlz4, lz4cat - Compress or decompress .lz4 files +================================================================ + +SYNOPSIS +-------- + +`lz4` [*OPTIONS*] [-|INPUT-FILE] <OUTPUT-FILE> + +`unlz4` is equivalent to `lz4 -d` + +`lz4cat` is equivalent to `lz4 -dcfm` + +When writing scripts that need to decompress files, +it is recommended to always use the name `lz4` with appropriate arguments +(`lz4 -d` or `lz4 -dc`) instead of the names `unlz4` and `lz4cat`. + + +DESCRIPTION +----------- + +`lz4` is an extremely fast lossless compression algorithm, +based on **byte-aligned LZ77** family of compression scheme. +`lz4` offers compression speeds of 400 MB/s per core, linearly scalable with +multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the `.lz4` format. + +### Difference between lz4 and gzip + +`lz4` supports a command line syntax similar _but not identical_ to `gzip(1)`. +Differences are : + + * `lz4` preserves original files + * `lz4` compresses a single file by default (see `-m` for multiple files) + * `lz4 file1 file2` means : compress file1 _into_ file2 + * `lz4 file.lz4` will default to decompression (use `-z` to force compression) + * `lz4` shows real-time notification statistics + during compression or decompression of a single file + (use `-q` to silent them) + * If no destination name is provided, result is sent to `stdout` + _except if stdout is the console_. + * If no destination name is provided, __and__ if `stdout` is the console, + `file` is compressed into `file.lz4`. + * As a consequence of previous rules, note the following example : + `lz4 file | consumer` sends compressed data to `consumer` through `stdout`, + hence it does _not_ create `file.lz4`. + +Default behaviors can be modified by opt-in commands, detailed below. + + * `lz4 -m` makes it possible to provide multiple input filenames, + which will be compressed into files using suffix `.lz4`. + Progress notifications are also disabled by default (use `-v` to enable them). + This mode has a behavior which more closely mimics `gzip` command line, + with the main remaining difference being that source files are preserved by default. + * Similarly, `lz4 -m -d` can decompress multiple `*.lz4` files. + * It's possible to opt-in to erase source files + on successful compression or decompression, using `--rm` command. + * Consequently, `lz4 -m --rm` behaves the same as `gzip`. + +### Concatenation of .lz4 files + +It is possible to concatenate `.lz4` files as is. +`lz4` will decompress such files as if they were a single `.lz4` file. +For example: + lz4 file1 > foo.lz4 + lz4 file2 >> foo.lz4 + +then + lz4cat foo.lz4 + +is equivalent to : + cat file1 file2 + + +OPTIONS +------- + +### Short commands concatenation + +In some cases, some options can be expressed using short command `-x` +or long command `--long-word`. +Short commands can be concatenated together. +For example, `-d -c` is equivalent to `-dc`. +Long commands cannot be concatenated. +They must be clearly separated by a space. + +### Multiple commands + +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. + +### Operation mode + +* `-z` `--compress`: + Compress. + This is the default operation mode when no operation mode option is + specified, no other operation mode is implied from the command name + (for example, `unlz4` implies `--decompress`), + nor from the input file name + (for example, a file extension `.lz4` implies `--decompress` by default). + `-z` can also be used to force compression of an already compressed + `.lz4` file. + +* `-d` `--decompress` `--uncompress`: + Decompress. + `--decompress` is also the default operation when the input filename has an + `.lz4` extension. + +* `-t` `--test`: + Test the integrity of compressed `.lz4` files. + The decompressed data is discarded. + No files are created nor removed. + +* `-b#`: + Benchmark mode, using `#` compression level. + +### Operation modifiers + +* `-#`: + Compression level, with # being any value from 1 to 16. + Higher values trade compression speed for compression ratio. + Values above 16 are considered the same as 16. + Recommended values are 1 for fast compression (default), + and 9 for high compression. + Speed/compression trade-off will vary depending on data to compress. + Decompression speed remains fast at all settings. + +* `-f` `--[no-]force`: + This option has several effects: + + If the target file already exists, overwrite it without prompting. + + When used with `--decompress` and `lz4` cannot recognize the type of + the source file, copy the source file as is to standard output. + This allows `lz4cat --force` to be used like `cat (1)` for files + that have not been compressed with `lz4`. + +* `-c` `--stdout` `--to-stdout`: + Force write to standard output, even if it is the console. + +* `-m` `--multiple`: + Multiple input files. + Compressed file names will be appended a `.lz4` suffix. + This mode also reduces notification level. + `lz4 -m` has a behavior equivalent to `gzip -k` + (it preserves source files by default). + +* `-r` : + operate recursively on directories. + This mode also sets `-m` (multiple input files). + +* `-B#`: + Block size \[4-7\](default : 7)<br/> + `-B4`= 64KB ; `-B5`= 256KB ; `-B6`= 1MB ; `-B7`= 4MB + +* `-BD`: + Block Dependency (improves compression ratio on small blocks) + +* `--[no-]frame-crc`: + Select frame checksum (default:enabled) + +* `--[no-]content-size`: + Header includes original size (default:not present)<br/> + Note : this option can only be activated when the original size can be + determined, hence for a file. It won't work with unknown source size, + such as stdin or pipe. + +* `--[no-]sparse`: + Sparse mode support (default:enabled on file, disabled on stdout) + +* `-l`: + Use Legacy format (typically for Linux Kernel compression)<br/> + Note : `-l` is not compatible with `-m` (`--multiple`) nor `-r` + +### Other options + +* `-v` `--verbose`: + Verbose mode + +* `-q` `--quiet`: + Suppress warnings and real-time statistics; + specify twice to suppress errors too + +* `-h` `-H` `--help`: + Display help/long help and exit + +* `-V` `--version`: + Display Version number and exit + +* `-k` `--keep`: + Preserve source files (default behavior) + +* `--rm` : + Delete source files on successful compression or decompression + + +### Benchmark mode + +* `-b#`: + Benchmark file(s), using # compression level + +* `-e#`: + Benchmark multiple compression levels, from b# to e# (included) + +* `-i#`: + Minimum evaluation in seconds \[1-9\] (default : 3) + + +BUGS +---- + +Report bugs at: https://github.com/lz4/lz4/issues + + +AUTHOR +------ + +Yann Collet diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 71378aa..cf91a99 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -39,17 +39,11 @@ /* #define ENABLE_LZ4C_LEGACY_OPTIONS */ -/************************************** -* Compiler Options -***************************************/ -/* cf. http://man7.org/linux/man-pages/man7/feature_test_macros.7.html */ -#define _XOPEN_VERSION 600 /* POSIX.2001, for fileno() within <stdio.h> on unix */ - - /**************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include <stdio.h> /* fprintf, getchar */ #include <stdlib.h> /* exit, calloc, free */ #include <string.h> /* strcmp, strlen */ @@ -59,26 +53,12 @@ #include "lz4.h" /* LZ4_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(_POSIX_SOURCE) || defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DJGPP__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ -# include <unistd.h> /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include <io.h> /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /***************************** * Constants ******************************/ #define COMPRESSOR_NAME "LZ4 command line interface" #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION_STRING, AUTHOR +#define WELCOME_MESSAGE "*** %s %i-bits v%s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_versionString(), AUTHOR #define LZ4_EXTENSION ".lz4" #define LZ4CAT "lz4cat" #define UNLZ4 "unlz4" @@ -129,19 +109,20 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output *****************************/ static int usage(const char* exeName) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "Usage : \n"); + DISPLAY( " %s [arg] [input] [output] \n", exeName); DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); + DISPLAY( "input : a filename \n"); DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); + DISPLAY( "Arguments : \n"); DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); + DISPLAY( " -z : force compression \n"); DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -k : preserve source files(s) (default) \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -h/-H : display help/long help and exit \n"); return 0; } @@ -151,33 +132,33 @@ static int usage_advanced(const char* exeName) usage(exeName); DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -V : display Version number and exit \n"); + DISPLAY( " -v : verbose mode \n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); #ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); + DISPLAY( " -r : operate recursively on directories (sets also -m) \n"); #endif DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7] (default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -B# : Block size [4-7] (default : 7) \n"); + DISPLAY( " -BD : Block dependency (improve compression ratio) \n"); /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); + DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [4-7] (default: 7)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s) \n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+] \n"); + DISPLAY( " or predefined block size [4-7] (default: 7) \n"); #if defined(ENABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); + DISPLAY( "Legacy arguments : \n"); + DISPLAY( " -c0 : fast compression \n"); + DISPLAY( " -c1 : high compression \n"); + DISPLAY( " -hc : high compression \n"); DISPLAY( " -y : overwrite output without prompting \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ EXTENDED_HELP; @@ -207,7 +188,7 @@ static int usage_longhelp(const char* exeName) DISPLAY( "Compression levels : \n"); DISPLAY( "---------------------\n"); DISPLAY( "-0 ... -2 => Fast compression, all identicals\n"); - DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_MAX_CLEVEL); + DISPLAY( "-3 ... -%d => High compression; higher number == more compression but slower\n", LZ4HC_CLEVEL_MAX); DISPLAY( "\n"); DISPLAY( "stdin, stdout and the console : \n"); DISPLAY( "--------------------------------\n"); @@ -259,6 +240,13 @@ static void waitEnter(void) (void)getchar(); } +static const char* lastNameFromPath(const char* path) +{ + const char* name = strrchr(path, '/'); + if (name==NULL) name = strrchr(path, '\\'); /* windows */ + if (name==NULL) return path; + return name+1; +} /*! readU32FromChar() : @return : unsigned integer value reach from input in `char` format @@ -293,7 +281,7 @@ int main(int argc, const char** argv) const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); - const char* const exeName = argv[0]; + const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; char* fileNamesBuf = NULL; @@ -305,12 +293,14 @@ int main(int argc, const char** argv) DISPLAY("Allocation error : not enough memory \n"); return 1; } + inFileNames[0] = stdinmark; LZ4IO_setOverwrite(0); /* lz4cat predefined behavior */ if (!strcmp(exeName, LZ4CAT)) { mode = om_decompress; LZ4IO_setOverwrite(1); + LZ4IO_setRemoveSrcFile(0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; @@ -459,7 +449,7 @@ int main(int argc, const char** argv) break; #ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ + /* recursive */ case 'r': recursive=1; /* without break */ #endif /* Treat non-option args as input files. See https://code.google.com/p/lz4/issues/detail?id=151 */ @@ -473,7 +463,7 @@ int main(int argc, const char** argv) iters = readU32FromChar(&argument); argument--; BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ } break; @@ -508,6 +498,18 @@ int main(int argc, const char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif +#ifdef _FILE_OFFSET_BITS + DISPLAYLEVEL(4, "_FILE_OFFSET_BITS defined: %ldL\n", (long) _FILE_OFFSET_BITS); +#endif if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); if (multiple_inputs) { @@ -546,9 +548,12 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } + /* if input==stdin and no output defined, stdout becomes default output */ + if (!strcmp(input_filename, stdinmark) && !output_filename) + output_filename = stdoutmark; /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) { + while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ size_t const inSize = strlen(input_filename); @@ -584,17 +589,18 @@ int main(int argc, const char** argv) } /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { DISPLAYLEVEL(1, "refusing to write to console without -c\n"); exit(1); } - /* Downgrade notification level in stdout and multiple file mode */ if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); diff --git a/programs/lz4io.c b/programs/lz4io.c index bc2ba95..640c76d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -30,24 +30,26 @@ - The license of this source file is GPLv2. */ -/************************************** -* Compiler Options -**************************************/ -#define _LARGE_FILES /* Large file support on 32-bits AIX */ -#define _FILE_OFFSET_BITS 64 /* off_t width */ -#define _LARGEFILE_SOURCE +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif + /***************************** * Includes *****************************/ -#include "util.h" /* Compiler options, UTIL_getFileStat */ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ #include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ +#include <string.h> /* strerror, strcmp, strlen */ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ @@ -57,30 +59,14 @@ #include "lz4frame.h" -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include <fcntl.h> /* _O_BINARY */ -# include <io.h> /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include <winioctl.h> /* FSCTL_SET_SPARSE */ -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || (defined(__APPLE__) && defined(__MACH__)) -# define fseek fseeko -# endif -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +#endif +#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko #endif @@ -591,12 +577,15 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t ress; + + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + ress = LZ4IO_createCResources(); /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } + if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); if (dstFileName==NULL) { LZ4IO_freeCResources(ress); return ifntSize; } } strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); @@ -660,8 +649,10 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer storedSkips += (unsigned)(nb0T * sizeT); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + errno = 0; + { int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); + } storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; @@ -1022,7 +1013,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz size_t const suffixSize = strlen(suffix); dRess_t ress = LZ4IO_createDResources(); - if (outFileName==NULL) exit(1); /* not enough memory */ + if (outFileName==NULL) return ifntSize; /* not enough memory */ ress.dstFile = LZ4IO_openDstFile(stdoutmark); for (i=0; i<ifntSize; i++) { @@ -1032,7 +1023,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); continue; } - if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) exit(1); } + if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } if (ifnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) { DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); skippedFiles++; diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 0000000..f1040c0 --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,143 @@ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include <unistd.h> /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include <unistd.h> /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include <io.h> /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif + + +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include <fcntl.h> /* _O_BINARY */ +# include <io.h> /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include <winioctl.h> /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ diff --git a/programs/util.h b/programs/util.h index 1ad61bc..f3ff1b2 100644 --- a/programs/util.h +++ b/programs/util.h @@ -15,11 +15,8 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -27,70 +24,49 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif /*-**************************************** * Dependencies ******************************************/ -#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */ -#include <stdio.h> /* fprintf */ -#include <string.h> /* strerr, strlen, memcpy */ -#include <stddef.h> /* ptrdiff_t */ -#include <sys/types.h> /* stat, utime */ -#include <sys/stat.h> /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include <stdlib.h> /* malloc */ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <stdio.h> /* fprintf */ +#include <sys/types.h> /* stat, utime */ +#include <sys/stat.h> /* stat */ #if defined(_MSC_VER) - #include <sys/utime.h> /* utime */ - #include <io.h> /* _chmod */ +# include <sys/utime.h> /* utime */ +# include <io.h> /* _chmod */ #else - #include <unistd.h> /* chown, stat */ - #include <utime.h> /* utime */ +# include <unistd.h> /* chown, stat */ +# include <utime.h> /* utime */ #endif -#include <time.h> /* time */ +#include <time.h> /* time */ #include <errno.h> -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; #else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; #endif @@ -102,7 +78,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include <unistd.h> # include <sys/resource.h> /* setpriority */ # include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -112,7 +88,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -124,27 +100,27 @@ extern "C" { #endif -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline #else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif /*-**************************************** @@ -307,7 +283,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -319,7 +295,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -356,16 +332,16 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; } -#elif (defined(__APPLE__) && defined(__MACH__)) || \ - ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include <dirent.h> /* opendir, readdir */ +# include <string.h> /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { @@ -427,7 +403,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (lz4 compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } @@ -443,16 +419,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) { if (!UTIL_isDirectory(inputNames[i])) { - size_t len = strlen(inputNames[i]); + size_t const len = strlen(inputNames[i]); if (buf + pos + len >= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -474,8 +449,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i<nbFiles; i++) - { + for (i=0, pos=0; i<nbFiles; i++) { fileTable[i] = buf + pos; pos += strlen(fileTable[i]) + 1; } |