From 9fd92def420879e7d5c7c9fb9a9abd2fbd54bb7a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Jan 2015 05:36:58 +0100 Subject: Added : Visual project directory --- .gitattributes | 5 + visual/2012/fuzzer/fuzzer.vcxproj | 159 ++++++++++++++++++++++++++++ visual/2012/fuzzer/fuzzer.vcxproj.filters | 42 ++++++++ visual/2012/fuzzer/fuzzer.vcxproj.user | 4 + visual/2012/lz4.sln | 46 +++++++++ visual/2012/lz4.v11.suo | Bin 0 -> 20992 bytes visual/2012/lz4/lz4.vcxproj | 166 ++++++++++++++++++++++++++++++ visual/2012/lz4/lz4.vcxproj.filters | 63 ++++++++++++ visual/2012/lz4/lz4.vcxproj.user | 4 + 9 files changed, 489 insertions(+) create mode 100644 visual/2012/fuzzer/fuzzer.vcxproj create mode 100644 visual/2012/fuzzer/fuzzer.vcxproj.filters create mode 100644 visual/2012/fuzzer/fuzzer.vcxproj.user create mode 100644 visual/2012/lz4.sln create mode 100644 visual/2012/lz4.v11.suo create mode 100644 visual/2012/lz4/lz4.vcxproj create mode 100644 visual/2012/lz4/lz4.vcxproj.filters create mode 100644 visual/2012/lz4/lz4.vcxproj.user diff --git a/.gitattributes b/.gitattributes index 17db2ec..b9d54fb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,8 @@ # Denote files that should not be modified. *.odt binary *.png binary +# Visual Studio +*.sln binary +*.suo binary +*.vcxproj* binary + diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj new file mode 100644 index 0000000..9ba4fab --- /dev/null +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -0,0 +1,159 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {18B9F1A7-9C66-4352-898B-30804DADE0FD} + Win32Proj + fuzzer + + + + Application + true + v110 + Unicode + + + Application + true + v110 + Unicode + + + Application + false + v110 + true + Unicode + + + Application + false + v110 + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.filters b/visual/2012/fuzzer/fuzzer.vcxproj.filters new file mode 100644 index 0000000..81002e9 --- /dev/null +++ b/visual/2012/fuzzer/fuzzer.vcxproj.filters @@ -0,0 +1,42 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + \ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.user b/visual/2012/fuzzer/fuzzer.vcxproj.user new file mode 100644 index 0000000..7cbb321 --- /dev/null +++ b/visual/2012/fuzzer/fuzzer.vcxproj.user @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/visual/2012/lz4.sln b/visual/2012/lz4.sln new file mode 100644 index 0000000..2d07075 --- /dev/null +++ b/visual/2012/lz4.sln @@ -0,0 +1,46 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2012 for Windows Desktop +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Mixed Platforms = Debug|Mixed Platforms + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Mixed Platforms = Release|Mixed Platforms + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.Build.0 = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.ActiveCfg = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.Build.0 = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.Build.0 = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.ActiveCfg = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.Build.0 = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/visual/2012/lz4.v11.suo b/visual/2012/lz4.v11.suo new file mode 100644 index 0000000..8321984 Binary files /dev/null and b/visual/2012/lz4.v11.suo differ diff --git a/visual/2012/lz4/lz4.vcxproj b/visual/2012/lz4/lz4.vcxproj new file mode 100644 index 0000000..5540d5d --- /dev/null +++ b/visual/2012/lz4/lz4.vcxproj @@ -0,0 +1,166 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {E30329AC-0057-4FE0-8FDA-7F650D398C4C} + Win32Proj + lz4 + + + + Application + true + v110 + Unicode + + + Application + true + v110 + Unicode + + + Application + false + v110 + true + Unicode + + + Application + false + v110 + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/visual/2012/lz4/lz4.vcxproj.filters b/visual/2012/lz4/lz4.vcxproj.filters new file mode 100644 index 0000000..fbe688b --- /dev/null +++ b/visual/2012/lz4/lz4.vcxproj.filters @@ -0,0 +1,63 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + \ No newline at end of file diff --git a/visual/2012/lz4/lz4.vcxproj.user b/visual/2012/lz4/lz4.vcxproj.user new file mode 100644 index 0000000..7cbb321 --- /dev/null +++ b/visual/2012/lz4/lz4.vcxproj.user @@ -0,0 +1,4 @@ + + + + \ No newline at end of file -- cgit v0.12 From 8f4e201a06109a4be4494aa5b024b3b4077e9d0d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Jan 2015 18:03:17 +0100 Subject: Fix : lz4frame.h within uninstaller --- lib/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Makefile b/lib/Makefile index faffb39..dcb481f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -112,6 +112,7 @@ uninstall: @[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 libraries successfully uninstalled endif -- cgit v0.12 From 41b6ed3c5bc7f0c85f1bfe3abc940d9b74581e7d Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Sat, 14 Feb 2015 12:48:11 +0900 Subject: Replace fseek with _fseeki64 to avoid MSVC's 2GiB barrier Since MSVC fseek()'s SEEK_CUR mode has 2GiB barrier, our fseek() calling in program/lz4io.c/selectDecoder() will fail for large (>2GiB) .lz4 file. This commit just replace fseek with _fseeki64 by macro. Second argument is automatically casted to __int64 (signed 64bit integer). Other things are completely same. --- programs/lz4io.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/lz4io.c b/programs/lz4io.c index fa1f0f9..023824e 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -71,6 +71,9 @@ # ifdef __MINGW32__ int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ # endif +# if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -- cgit v0.12 From e68d1c98c1b4f036f3345ca56d8649b34305e9cb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 2 Mar 2015 00:11:34 +0100 Subject: restored lz4 hc compression ratio --- programs/Makefile | 3 +- programs/fullbench.c | 172 +++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 148 insertions(+), 27 deletions(-) mode change 100644 => 100755 programs/fullbench.c diff --git a/programs/Makefile b/programs/Makefile index 543eb7c..076d3bf 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -20,7 +20,7 @@ # # You can contact the author at : # - LZ4 source repository : http://code.google.com/p/lz4/ -# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c +# - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c # ########################################################################## # lz4 : Command Line Utility, supporting gzip-like arguments # lz4c : CLU, supporting also legacy lz4demo arguments @@ -143,6 +143,7 @@ test-lz4: lz4 datagen ./datagen -g16KB | ./lz4 -9 | ./lz4 -vdq > $(VOID) ./datagen | ./lz4 | ./lz4 -vdq > $(VOID) ./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID) + ./datagen -g17M | ./lz4 -9v | ./lz4 -vdq > $(VOID) ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID) ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID) # test frame concatenation with null-length frame diff --git a/programs/fullbench.c b/programs/fullbench.c old mode 100644 new mode 100755 index b785924..756357a --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -24,12 +24,12 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -//************************************** -// Compiler Options -//************************************** -// Disable some Visual warning messages +/************************************** +* Compiler Options +**************************************/ +/* Disable some Visual warning messages */ #define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE // VS2005 +#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ // Unix Large Files support (>4GB) #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions @@ -45,9 +45,9 @@ #endif -//************************************** -// Includes -//************************************** +/************************************** +* Includes +**************************************/ #include // malloc #include // fprintf, fopen, ftello64 #include // stat64 @@ -68,10 +68,10 @@ #include "xxhash.h" -//************************************** -// Compiler Options -//************************************** -// S_ISREG & gettimeofday() are not supported by MSVC +/************************************** +* Compiler Options +**************************************/ +/* S_ISREG & gettimeofday() are not supported by MSVC */ #if !defined(S_ISREG) # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif @@ -82,9 +82,9 @@ #endif -//************************************** -// Basic Types -//************************************** +/************************************** +* Basic Types +**************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 # include typedef uint8_t BYTE; @@ -101,9 +101,9 @@ #endif -//**************************** -// Constants -//**************************** +/************************************** +* Constants +**************************************/ #define PROGRAM_DESCRIPTION "LZ4 speed analyzer" #ifndef LZ4_VERSION # define LZ4_VERSION "" @@ -122,9 +122,9 @@ #define ALL_DECOMPRESSORS 0 -//************************************** -// Local structures -//************************************** +/************************************** +* Local structures +**************************************/ struct chunkParameters { U32 id; @@ -135,9 +135,9 @@ struct chunkParameters }; -//************************************** -// MACRO -//************************************** +/************************************** +* MACRO +**************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__) @@ -251,8 +251,127 @@ static U64 BMK_GetFileSize(char* infilename) /********************************************************* - Benchmark function +* Benchmark function *********************************************************/ +#ifdef __SSSE3__ + +#include + +/* Idea proposed by Terje Mathisen */ +static BYTE stepSize16[17] = {16,16,16,15,16,15,12,14,16,9,10,11,12,13,14,15,16}; +static __m128i replicateTable[17] = { + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, + {0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1}, + {0,1,2,0,1,2,0,1,2,0,1,2,0,1,2,0}, + {0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3}, + {0,1,2,3,4,0,1,2,3,4,0,1,2,3,4,0}, + {0,1,2,3,4,5,0,1,2,3,4,5,0,1,2,3}, + {0,1,2,3,4,5,6,0,1,2,3,4,5,6,0,1}, + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, + {0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6}, + {0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}, + {0,1,2,3,4,5,6,7,8,9,10,0,1,2,3,4}, + {0,1,2,3,4,5,6,7,8,9,10,11,0,1,2,3}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,0,1,2}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,0,1}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}}; +static BYTE stepSize32[17] = {32,32,32,30,32,30,30,28,32,27,30,22,24,26,28,30,16}; +static __m128i replicateTable2[17] = { + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, + {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, + {0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1}, + {1,2,0,1,2,0,1,2,0,1,2,0,1,2,0,1}, + {0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3}, + {1,2,3,4,0,1,2,3,4,0,1,2,3,4,0,1}, + {4,5,0,1,2,3,4,5,0,1,2,3,4,5,0,1}, + {2,3,4,5,6,0,1,2,3,4,5,6,0,1,2,3}, + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, + {7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4}, + {6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1}, + {5,6,7,8,9,10,0,1,2,3,4,5,6,7,8,9}, + {4,5,6,7,8,9,10,11,0,1,2,3,4,5,6,7}, + {3,4,5,6,7,8,9,10,11,12,0,1,2,3,4,5}, + {2,3,4,5,6,7,8,9,10,11,12,13,0,1,2,3}, + {1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,1}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}}; + +U32 lz4_decode_sse(BYTE* dest, BYTE* src, U32 srcLength) +{ + BYTE* d = dest, *e = src+srcLength; + unsigned token, lit_len, mat_len; + __m128i a; + BYTE* dstore, *msrc; + + if (!srcLength) return 0; + goto start; + + do { + U32 step; + unsigned mat_offset = src[0] + (src[1] << 8); + src += 2; + msrc = d - mat_offset; + if (mat_len == 15) { + do { + token = *src++; + mat_len += token; + } while (token == 255); + } + mat_len += 4; + + dstore = d; + d += mat_len; + + if (mat_offset <= 16) + { // Bulk store only! + __m128i a2; + a = _mm_loadu_si128((const __m128i *)msrc); + a2 = _mm_shuffle_epi8(a, replicateTable2[mat_offset]); + a = _mm_shuffle_epi8(a, replicateTable[mat_offset]); + step = stepSize32[mat_offset]; + do { + _mm_storeu_si128((__m128i *)dstore, a); + _mm_storeu_si128((__m128i *)(dstore+16), a2); + dstore += step; + } while (dstore < d); + } + else + { + do + { + a = _mm_loadu_si128((const __m128i *)msrc); + _mm_storeu_si128((__m128i *)dstore, a); + msrc += sizeof(a); + dstore += sizeof(a); + } while (dstore < d); + } +start: + token = *src++; + lit_len = token >> 4; + mat_len = token & 15; + if (token >= 0xf0) { // lit_len == 15 + do { + token = *src++; + lit_len += token; + } while (token == 255); + } + dstore = d; + msrc = src; + d += lit_len; + src += lit_len; + do { + a = _mm_loadu_si128((const __m128i *)msrc); + _mm_storeu_si128((__m128i *)dstore, a); + msrc += sizeof(a); + dstore += sizeof(a); + } while (dstore < d); + } while (src < e); + + return (U32)(d-dest); +} +#endif // __SSSE3__ + static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) { @@ -345,6 +464,7 @@ static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) { (void)inSize; + //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize); LZ4_decompress_fast(in, out, outSize); return outSize; } @@ -677,7 +797,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); - // CRC Checking + /* CRC Checking */ crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } } -- cgit v0.12 From fa27d236b4f3fc9938478e4464b63953d6429d4b Mon Sep 17 00:00:00 2001 From: kyle Date: Sun, 1 Mar 2015 19:00:40 -0600 Subject: Added support for multiple input files to act more like other compressors. For example: gzip file1 file2 file3. You can now do: lz4 [args] -m file1 file2 file3. Fixes 151. --- programs/lz4cli.c | 142 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 58 deletions(-) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0da5dce..7524947 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -156,6 +156,7 @@ static int usage(void) DISPLAY( " -z : force compression\n"); DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -m : allow mulitple input files (implies automatic output filenames)"); return 0; } @@ -265,10 +266,12 @@ int main(int argc, char** argv) legacy_format=0, forceStdout=0, forceCompress=0, - main_pause=0; + main_pause=0, + multiple_inputs=0; char* input_filename=0; char* output_filename=0; char* dynNameSpace=0; + char *input_filenames[argc]; char nullOutput[] = NULL_OUTPUT; char extension[] = LZ4_EXTENSION; int blockSize; @@ -284,6 +287,7 @@ int main(int argc, char** argv) /* command switches */ for(i=1; i='1') && (argument[1] <='9')) @@ -413,10 +420,13 @@ int main(int argc, char** argv) continue; } - /* first provided filename is input */ + /* Input files are a wonderful thing. Store in *input_filenames[] if -m is used. */ + if (multiple_inputs) { input_filenames[i]=argument; continue; } + + /* Store first non-option arg in input_filename to preserve original cli logic. */ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - /* second provided filename is output */ + /* Second non-option arg in output_filename to preserve original cli logic. */ if (!output_filename) { output_filename=argument; @@ -425,74 +435,90 @@ int main(int argc, char** argv) } } - DISPLAYLEVEL(3, WELCOME_MESSAGE); - if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); + /* If we are not using multiple_inputs, clear *input_filesnames[] (just in case) and set element 0 to input_filename. */ + if (!multiple_inputs) + { + for (i=0; i use stdin */ - if(!input_filename) { input_filename=stdinmark; } + /* Loop through all the input_filenames[], which should only be 1 if we're using the old (original, non -m) cli method. + This is a shamelessly-stolen loop over the original logic. */ + for (i=0; i>10); - /* Check if input or output are defined as console; trigger an error in this case */ - if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + /* No input filename ==> use stdin */ + if(!input_filename) { input_filename=stdinmark; } - /* Check if benchmark is selected */ - if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + /* Check if input or output are defined as console; trigger an error in this case */ + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); - /* No output filename ==> try to select one automatically (when possible) */ - while (!output_filename) - { - if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ - if ((!decode) && !(forceCompress)) /* auto-determine compression or decompression, based on file extension */ - { - size_t l = strlen(input_filename); - if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; - } - if (!decode) /* compression to file */ - { - size_t l = strlen(input_filename); - dynNameSpace = (char*)calloc(1,l+5); - output_filename = dynNameSpace; - strcpy(output_filename, input_filename); - strcpy(output_filename+l, LZ4_EXTENSION); - DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); - break; - } - /* decompression to file (automatic name will work only if input filename has correct format extension) */ + /* Check if benchmark is selected */ + if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + + /* No output filename ==> try to select one automatically (when possible) */ + while (!output_filename) { - size_t outl; - size_t inl = strlen(input_filename); - dynNameSpace = (char*)calloc(1,inl+1); - output_filename = dynNameSpace; - strcpy(output_filename, input_filename); - outl = inl; - if (inl>4) - while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } - DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); + if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ + if ((!decode) && !(forceCompress)) /* auto-determine compression or decompression, based on file extension */ + { + size_t l = strlen(input_filename); + if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; + } + if (!decode) /* compression to file */ + { + size_t l = strlen(input_filename); + dynNameSpace = (char*)calloc(1,l+5); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + strcpy(output_filename+l, LZ4_EXTENSION); + DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); + break; + } + /* decompression to file (automatic name will work only if input filename has correct format extension) */ + { + size_t outl; + size_t inl = strlen(input_filename); + dynNameSpace = (char*)calloc(1,inl+1); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + outl = inl; + if (inl>4) + while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } + DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); + } } - } - /* Check if output is defined as console; trigger an error in this case */ - if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); + /* Check if output is defined as console; trigger an error in this case */ + if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - /* No warning message in pure pipe mode (stdin + stdout) */ - if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + /* No warning message in pure pipe mode (stdin + stdout) */ + if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; - /* IO Stream/File */ - LZ4IO_setNotificationLevel(displayLevel); - if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename); - else - { - /* compression is default action */ - if (legacy_format) - { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); - } + /* IO Stream/File */ + LZ4IO_setNotificationLevel(displayLevel); + if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename); else { - DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + /* compression is default action */ + if (legacy_format) + { + DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); + LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); + } + else + { + DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + } } + /* Clear the output_filename here in case there are other files to process. */ + output_filename=0; } if (main_pause) waitEnter(); -- cgit v0.12 From 97679fa5a17d79f87c0a2d1e2d44c3215d48958b Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 15:46:02 +0900 Subject: Google Code Issue 155: lz4 cli should support sparse file https://code.google.com/p/lz4/issues/detail?id=155 This is experimental implementation. Just a proof of concept. It works Linux and Windows. # Build To build experimental version, define 'LZ4IO_ENABLE_SPARSE_FILE' like the following command : make lz4programs 'CFLAGS=-O3 -DLZ4IO_ENABLE_SPARSE_FILE=1' ./programs/lz4 -h You will see "EXPERIMENTAL_SPARSE_FILE" as lz4 revision : "*** LZ4 command line interface 64-bits EXPERIMENTAL_SPARSE_FILE, by Yann Collet (...) ***" # Experiment This experimental version adds option "-x" for sparse file for decompression. You can use this option like this : ./programs/lz4 -9 -f my-file ./programs/lz4 -d -f -x my-file.lz4 my-file.lz4.out cmp my-file my-file.lz4.out --- programs/lz4cli.c | 17 +++++++++++++++ programs/lz4io.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ programs/lz4io.h | 5 +++++ 3 files changed, 87 insertions(+) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 0da5dce..5b54143 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -83,6 +83,11 @@ /***************************** * Constants ******************************/ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +# undef LZ4_VERSION +# define LZ4_VERSION "EXPERIMENTAL_SPARSE_FILE" +#endif + #define COMPRESSOR_NAME "LZ4 command line interface" #ifndef LZ4_VERSION # define LZ4_VERSION "r126" @@ -186,6 +191,10 @@ static int usage_advanced(void) DISPLAY( " -y : overwrite output without prompting \n"); DISPLAY( " -s : suppress warnings \n"); #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + DISPLAY( "Experimental : Sparse file\n"); + DISPLAY( " -x : enable sparse file\n"); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ EXTENDED_HELP; return 0; } @@ -276,6 +285,9 @@ int main(int argc, char** argv) /* Init */ programName = argv[0]; LZ4IO_setOverwrite(0); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + LZ4IO_setSparseFile(0); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); /* lz4cat behavior */ @@ -403,6 +415,11 @@ int main(int argc, char** argv) /* Pause at the end (hidden option) */ case 'p': main_pause=1; BMK_SetPause(); break; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + /* Experimental : Enable sparse file */ + case 'x': LZ4IO_setSparseFile(1); break; +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* Specific commands for customized versions */ EXTENDED_ARGUMENTS; diff --git a/programs/lz4io.c b/programs/lz4io.c index 023824e..4e583f6 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -76,10 +76,18 @@ # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +# if defined(LZ4IO_ENABLE_SPARSE_FILE) +# include +# define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0) +# define fseek _fseeki64 +# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #else # include /* isatty */ # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +# if defined(LZ4IO_ENABLE_SPARSE_FILE) +# define SET_SPARSE_FILE_MODE(file) +# endif /* LZ4IO_ENABLE_SPARSE_FILE */ #endif @@ -133,6 +141,9 @@ static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; static int blockChecksum = 0; static int streamChecksum = 1; static int blockIndependence = 1; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +static int sparseFile = 0; +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ static const int minBlockSizeID = 4; static const int maxBlockSizeID = 7; @@ -174,6 +185,28 @@ int LZ4IO_setOverwrite(int yes) return overwrite; } +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +/* Default setting : sparseFile = 0; (Disable) + return : sparse file mode (0:Disable / 1:Enable) */ +int LZ4IO_setSparseFile(int yes) +{ + sparseFile = yes; + return sparseFile; +} + +static int isSparse(const void* p, size_t size) +{ + const char* p8 = p; + for(; size; --size) { + if(*p8 != 0) { + return 0; + } + ++p8; + } + return 1; +} +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* blockSizeID : valid values : 4-5-6-7 */ int LZ4IO_setBlockSizeID(int bsid) { @@ -539,6 +572,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) LZ4F_decompressionContext_t ctx; LZ4F_errorCode_t errorCode; LZ4F_frameInfo_t frameInfo; +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + size_t sparsePending = 0; +#endif /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -580,9 +616,30 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) filesize += decodedBytes; /* Write Block */ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if(sparseFile) { + if(isSparse(outBuff, decodedBytes)) { + sparsePending += decodedBytes; + continue; + } + if(sparsePending > 0) { + fseek(foutput, sparsePending, SEEK_CUR); + sparsePending = 0; + } + } +#endif sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if(sparseFile) { + if(sparsePending > 0) { + fseek(foutput, sparsePending-1, SEEK_CUR); + fputc(0, foutput); + sparsePending = 0; + } + } +#endif /* Free */ free(inBuff); @@ -645,6 +702,14 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename) start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + if (sparseFile!=0 && foutput!=0) + { + DISPLAY("Experimental : Using sparse file\n"); + SET_SPARSE_FILE_MODE(foutput); + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + /* Loop over multiple streams */ do { diff --git a/programs/lz4io.h b/programs/lz4io.h index 7869a43..a0c6119 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -75,3 +75,8 @@ int LZ4IO_setStreamChecksumMode(int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); + +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +/* Default setting : 0 (sparseFile = 0; disable sparse file) */ +int LZ4IO_setSparseFile(int yes); +#endif -- cgit v0.12 From 4a5d92b1d9207b5d1309e1b21ecc1164cc14a149 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 16:13:23 +0900 Subject: Adjust coding style --- programs/lz4io.c | 65 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 4e583f6..9f80531 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -79,7 +79,9 @@ # if defined(LZ4IO_ENABLE_SPARSE_FILE) # include # define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0) -# define fseek _fseeki64 +# if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define fseek _fseeki64 +# endif # endif /* LZ4IO_ENABLE_SPARSE_FILE */ #else # include /* isatty */ @@ -197,13 +199,15 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { const char* p8 = p; - for(; size; --size) { - if(*p8 != 0) { - return 0; - } - ++p8; - } - return 1; + for(; size; --size) + { + if(*p8 != 0) + { + return 0; + } + ++p8; + } + return 1; } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ @@ -574,7 +578,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) LZ4F_frameInfo_t frameInfo; #if defined(LZ4IO_ENABLE_SPARSE_FILE) size_t sparsePending = 0; -#endif +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ /* init */ errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); @@ -617,29 +621,34 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) /* Write Block */ #if defined(LZ4IO_ENABLE_SPARSE_FILE) - if(sparseFile) { - if(isSparse(outBuff, decodedBytes)) { - sparsePending += decodedBytes; - continue; - } - if(sparsePending > 0) { - fseek(foutput, sparsePending, SEEK_CUR); - sparsePending = 0; - } - } -#endif + if(sparseFile) + { + if(isSparse(outBuff, decodedBytes)) + { + sparsePending += decodedBytes; + continue; + } + if(sparsePending > 0) + { + fseek(foutput, sparsePending, SEEK_CUR); + sparsePending = 0; + } + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } #if defined(LZ4IO_ENABLE_SPARSE_FILE) - if(sparseFile) { - if(sparsePending > 0) { - fseek(foutput, sparsePending-1, SEEK_CUR); - fputc(0, foutput); - sparsePending = 0; - } - } -#endif + if(sparseFile) + { + if(sparsePending > 0) + { + fseek(foutput, sparsePending-1, SEEK_CUR); + fputc(0, foutput); + sparsePending = 0; + } + } +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ /* Free */ free(inBuff); -- cgit v0.12 From 01a24afdcf867a901076eaa87434b3dcb39d526f Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 19:58:49 +0900 Subject: Improve isSparse() --- programs/lz4io.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/programs/lz4io.c b/programs/lz4io.c index 9f80531..b1067ac 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -62,6 +62,27 @@ #include "lz4frame.h" +/************************************** + Basic Types +**************************************/ +#if defined(LZ4IO_ENABLE_SPARSE_FILE) +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ + + /**************************** * OS-specific Includes *****************************/ @@ -198,6 +219,7 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { +#if 0 const char* p8 = p; for(; size; --size) { @@ -208,6 +230,30 @@ static int isSparse(const void* p, size_t size) ++p8; } return 1; +#else + const U64* p64 = (const U64*) p; + const char* p8 = (const char*) p; + const size_t n = size / sizeof(*p64); + size_t i; + + for (i = 0; i < n; ++i) + { + if (p64[i] != 0) + { + return 0; + } + } + + for(i = n * sizeof(*p64); i < size; ++i) + { + if (p8[i] != 0) + { + return 0; + } + } + + return 1; +#endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From b372f45093aa8d86f7a99b7dd5e2deaeffb10033 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 20:18:16 +0900 Subject: Add Neil's method to isSparse() --- programs/lz4io.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index b1067ac..c69275d 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -220,6 +220,7 @@ int LZ4IO_setSparseFile(int yes) static int isSparse(const void* p, size_t size) { #if 0 + /* naive */ const char* p8 = p; for(; size; --size) { @@ -230,7 +231,8 @@ static int isSparse(const void* p, size_t size) ++p8; } return 1; -#else +#elif 0 + /* xz method */ const U64* p64 = (const U64*) p; const char* p8 = (const char*) p; const size_t n = size / sizeof(*p64); @@ -253,6 +255,10 @@ static int isSparse(const void* p, size_t size) } return 1; +#else + /* Neil's */ + const char* buf = (const char*) p; + return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1); #endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From eed7952101fbb740018cd00d7d2958d192da8344 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 21:00:50 +0900 Subject: Add GNU coreutil's is_nul() method to isSparse() See original source http://git.savannah.gnu.org/cgit/coreutils.git/tree/src/system.h --- programs/lz4io.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index c69275d..7315d69 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -255,10 +255,24 @@ static int isSparse(const void* p, size_t size) } return 1; -#else +#elif 0 /* Neil's */ const char* buf = (const char*) p; return buf[0] == 0 && !memcmp(buf, buf + 1, size - 1); +#else + /* GNU Core Utilities : coreutils/src/system.h / is_nul() */ + const U64* wp = (const U64*) p; + const char* cbuf = (const char*) p; + const char* cp; + + // Find first nonzero *word*, or the word with the sentinel. + while(*wp++ == 0) ; + + // Find the first nonzero *byte*, or the sentinel. + cp = (const char*) (wp - 1); + while(*cp++ == 0) ; + + return cbuf + size < cp; #endif } #endif /* LZ4IO_ENABLE_SPARSE_FILE */ @@ -652,7 +666,12 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); inBuffSize = outBuffSize + 4; inBuff = (char*)malloc(inBuffSize); +#if defined(LZ4IO_ENABLE_SPARSE_FILE) + outBuff = (char*)malloc(outBuffSize+1); + outBuff[outBuffSize] = 1; /* sentinel */ +#else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); +#endif /* LZ4IO_ENABLE_SPARSE_FILE */ if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); /* Main Loop */ -- cgit v0.12 From de5c930c901539082c57193b390fa3666a732883 Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 21:26:32 +0900 Subject: Fix sentinel size miscalculation Since is_nul() style wild-comparing stride is U64, we must add sizeof(U64) to the sentinel. --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 7315d69..03d93af 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -667,7 +667,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) inBuffSize = outBuffSize + 4; inBuff = (char*)malloc(inBuffSize); #if defined(LZ4IO_ENABLE_SPARSE_FILE) - outBuff = (char*)malloc(outBuffSize+1); + outBuff = (char*)malloc(outBuffSize+sizeof(U64)); outBuff[outBuffSize] = 1; /* sentinel */ #else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); -- cgit v0.12 From dcdd628d5ec8a78401dbe277151e7654e300eafc Mon Sep 17 00:00:00 2001 From: Takayuki MATSUOKA Date: Thu, 12 Feb 2015 23:50:14 +0900 Subject: Fix sentinel bit pattern Set all bits of U64. --- programs/lz4io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/lz4io.c b/programs/lz4io.c index 03d93af..109db98 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -668,7 +668,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) inBuff = (char*)malloc(inBuffSize); #if defined(LZ4IO_ENABLE_SPARSE_FILE) outBuff = (char*)malloc(outBuffSize+sizeof(U64)); - outBuff[outBuffSize] = 1; /* sentinel */ + *(U64*) &outBuff[outBuffSize] = (U64) -1; /* sentinel */ #else /* LZ4IO_ENABLE_SPARSE_FILE */ outBuff = (char*)malloc(outBuffSize); #endif /* LZ4IO_ENABLE_SPARSE_FILE */ -- cgit v0.12 From 488029ec3b23b1e00a69f1ac3fcdd49d24cb7d3f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 7 Mar 2015 13:23:00 +0100 Subject: Updated : compress multiple files --- NEWS | 5 ++ programs/Makefile | 8 ++- programs/bench.c | 8 +-- programs/bench.h | 4 +- programs/lz4cli.c | 167 +++++++++++++++++++++++++++--------------------------- programs/lz4io.c | 33 +++++++++-- programs/lz4io.h | 8 ++- 7 files changed, 132 insertions(+), 101 deletions(-) diff --git a/NEWS b/NEWS index 8aeab1b..05ac6b5 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +r128: +New : command -m, to compress multiple files in a single command +Fixed : Restored lz4hc compression ratio (slightly lower since r124) +Added : Visual project directory + r126: New : lz4frame API is now integrated into liblz4 Fixed : GCC 4.9 bug on highest performance settings, reported by Greg Slazinski diff --git a/programs/Makefile b/programs/Makefile index 076d3bf..1070f40 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -31,7 +31,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE?= r126 +RELEASE?= r128 DESTDIR?= PREFIX ?= /usr @@ -158,6 +158,12 @@ test-lz4: lz4 datagen @rm *.test @echo frame concatenation test completed # test frame concatenation with null-length frame + @echo test multiple input files + @./datagen -s1 > file1 + @./datagen -s2 > file2 + @./datagen -s3 > file3 + ./lz4 -f -m file1 file2 file3 + @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 test-lz4c: lz4c datagen diff --git a/programs/bench.c b/programs/bench.c index 02e56c9..77120f2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -1,6 +1,6 @@ /* bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2014 + Copyright (C) Yann Collet 2012-2015 GPL v2 License This program is free software; you can redistribute it and/or modify @@ -215,7 +215,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(char* infilename) +static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) @@ -234,7 +234,7 @@ static U64 BMK_GetFileSize(char* infilename) * Public function **********************************************************/ -int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) +int BMK_benchFile(const char** fileNamesTable, int nbFiles, int cLevel) { int fileIdx=0; char* orig_buff; @@ -265,7 +265,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) while (fileIdx