diff options
| -rw-r--r-- | Makefile | 7 | ||||
| -rw-r--r-- | NEWS | 68 | ||||
| -rw-r--r-- | bench.c | 882 | ||||
| -rw-r--r-- | bench.h | 82 | ||||
| -rw-r--r-- | cmake/CMakeLists.txt | 90 | ||||
| -rw-r--r-- | fullbench.c | 1448 | ||||
| -rw-r--r-- | fuzzer.c | 583 | ||||
| -rw-r--r-- | lz4.1 | 169 | ||||
| -rw-r--r-- | lz4.c | 1687 | ||||
| -rw-r--r-- | lz4.h | 454 | ||||
| -rw-r--r-- | lz4cli.c | 2508 | ||||
| -rw-r--r-- | lz4hc.c | 1673 | ||||
| -rw-r--r-- | lz4hc.h | 268 | ||||
| -rw-r--r-- | xxhash.c | 950 | ||||
| -rw-r--r-- | xxhash.h | 328 |
15 files changed, 5710 insertions, 5487 deletions
@@ -30,14 +30,14 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ################################################################ -RELEASE=r109 +RELEASE=r110 DESTDIR= PREFIX=${DESTDIR}/usr BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man/man1 DISTRIBNAME=lz4-$(RELEASE).tar.gz CC=gcc -CFLAGS=-I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"v1.0.9\" +CFLAGS=-I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\" # Define *.exe as extension for Windows systems # ifeq ($(OS),Windows_NT) @@ -85,7 +85,8 @@ fullbench32: lz4.c lz4hc.c xxhash.c fullbench.c clean: @rm -f core *.o lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) + fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) \ + $(DISTRIBNAME) @echo Cleaning completed @@ -1,30 +1,38 @@ -r109 :
-lz4.c : corrected issue 98 (LZ4_compress_limitedOutput())
-Makefile : can specify version number from makefile
-
-r108 :
-lz4.c : corrected compression efficiency issue 97 in 64-bits chained mode (-BD) for streams > 4 GB (thanks Roman Strashkin for reporting)
-
-r107 :
-Makefile : support DESTDIR for staged installs. Thanks Jorge Aparicio.
-Makefile : make install installs both lz4 and lz4c (Jorge Aparicio)
-Makefile : removed -Wno-implicit-declaration compilation switch
-lz4cli.c : include <stduni.h> for isatty() (Luca Barbato)
-lz4.h : introduced LZ4_MAX_INPUT_SIZE constant (Shay Green)
-lz4.h : LZ4_compressBound() : unified macro and inline definitions (Shay Green)
-lz4.h : LZ4_decompressSafe_partial() : clarify comments (Shay Green)
-lz4.c : LZ4_compress() verify input size condition (Shay Green)
-bench.c : corrected a bug in free memory size evaluation
-cmake : install into bin/ directory (Richard Yao)
-cmake : check for just C compiler (Elan Ruusamae)
-
-r106 :
-Makefile : make dist modify text files in the package to respect Unix EoL convention
-lz4cli.c : corrected small display bug in HC mode
-
-r105 :
-Makefile : New install script and man page, contributed by Prasad Pandit
-lz4cli.c : Minor modifications, for easier extensibility
-COPYING : added license file
-LZ4_Streaming_Format.odt : modified file name to remove white space characters
-Makefile : .exe suffix now properly added only for Windows target
\ No newline at end of file +r110 : +lz4 & lz4hc : added capability to allocate state & stream state with custom allocator (issue 99) +fuzzer & fullbench : updated to test new functions +man : documented -l command (Legacy format, for Linux kernel compression) (issue 102) +cmake : improved version by Mika Attila, building programs and libraries (issue 100) +xxHash : updated to r33 +Makefile : clean also delete local package .tar.gz + +r109 : +lz4.c : corrected issue 98 (LZ4_compress_limitedOutput()) +Makefile : can specify version number from makefile + +r108 : +lz4.c : corrected compression efficiency issue 97 in 64-bits chained mode (-BD) for streams > 4 GB (thanks Roman Strashkin for reporting) + +r107 : +Makefile : support DESTDIR for staged installs. Thanks Jorge Aparicio. +Makefile : make install installs both lz4 and lz4c (Jorge Aparicio) +Makefile : removed -Wno-implicit-declaration compilation switch +lz4cli.c : include <stduni.h> for isatty() (Luca Barbato) +lz4.h : introduced LZ4_MAX_INPUT_SIZE constant (Shay Green) +lz4.h : LZ4_compressBound() : unified macro and inline definitions (Shay Green) +lz4.h : LZ4_decompressSafe_partial() : clarify comments (Shay Green) +lz4.c : LZ4_compress() verify input size condition (Shay Green) +bench.c : corrected a bug in free memory size evaluation +cmake : install into bin/ directory (Richard Yao) +cmake : check for just C compiler (Elan Ruusamae) + +r106 : +Makefile : make dist modify text files in the package to respect Unix EoL convention +lz4cli.c : corrected small display bug in HC mode + +r105 : +Makefile : New install script and man page, contributed by Prasad Pandit +lz4cli.c : Minor modifications, for easier extensibility +COPYING : added license file +LZ4_Streaming_Format.odt : modified file name to remove white space characters +Makefile : .exe suffix now properly added only for Windows target @@ -1,441 +1,441 @@ -/*
- bench.c - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2013
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-//**************************************
-// Compiler Options
-//**************************************
-// Disable some Visual warning messages
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE // VS2005
-
-// Unix Large Files support (>4GB)
-#define _FILE_OFFSET_BITS 64
-#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
-# define _LARGEFILE_SOURCE
-#elif ! defined(__LP64__) // No point defining Large file for 64 bit
-# define _LARGEFILE64_SOURCE
-#endif
-
-// S_ISREG & gettimeofday() are not supported by MSVC
-#if defined(_MSC_VER) || defined(_WIN32)
-# define BMK_LEGACY_TIMER 1
-#endif
-
-
-//**************************************
-// Includes
-//**************************************
-#include <stdlib.h> // malloc
-#include <stdio.h> // fprintf, fopen, ftello64
-#include <sys/types.h> // stat64
-#include <sys/stat.h> // stat64
-
-// Use ftime() if gettimeofday() is not available on your target
-#if defined(BMK_LEGACY_TIMER)
-# include <sys/timeb.h> // timeb, ftime
-#else
-# include <sys/time.h> // gettimeofday
-#endif
-
-#include "lz4.h"
-#define COMPRESSOR0 LZ4_compress
-#include "lz4hc.h"
-#define COMPRESSOR1 LZ4_compressHC
-#define DEFAULTCOMPRESSOR COMPRESSOR0
-
-#include "xxhash.h"
-
-
-//**************************************
-// Compiler specifics
-//**************************************
-#if !defined(S_ISREG)
-# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
-#endif
-
-// GCC does not support _rotl outside of Windows
-#if !defined(_WIN32)
-# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
-#endif
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-
-//**************************************
-// Constants
-//**************************************
-#define NBLOOPS 3
-#define TIMELOOP 2000
-
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-#define KNUTH 2654435761U
-#define MAX_MEM (2 GB - 64 MB)
-#define DEFAULT_CHUNKSIZE (4 MB)
-
-
-//**************************************
-// Local structures
-//**************************************
-struct chunkParameters
-{
- U32 id;
- char* origBuffer;
- char* compressedBuffer;
- int origSize;
- int compressedSize;
-};
-
-struct compressionParameters
-{
- int (*compressionFunction)(const char*, char*, int);
- int (*decompressionFunction)(const char*, char*, int);
-};
-
-
-//**************************************
-// MACRO
-//**************************************
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-
-
-
-//**************************************
-// Benchmark Parameters
-//**************************************
-static int chunkSize = DEFAULT_CHUNKSIZE;
-static int nbIterations = NBLOOPS;
-static int BMK_pause = 0;
-
-void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
-
-void BMK_SetNbIterations(int nbLoops)
-{
- nbIterations = nbLoops;
- DISPLAY("- %i iterations -\n", nbIterations);
-}
-
-void BMK_SetPause() { BMK_pause = 1; }
-
-
-//*********************************************************
-// Private functions
-//*********************************************************
-
-#if defined(BMK_LEGACY_TIMER)
-
-static int BMK_GetMilliStart()
-{
- // Based on Legacy ftime()
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)
- // Use GetMilliSpan to correct for rollover
- struct timeb tb;
- int nCount;
- ftime( &tb );
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
- return nCount;
-}
-
-#else
-
-static int BMK_GetMilliStart()
-{
- // Based on newer gettimeofday()
- // Use GetMilliSpan to correct for rollover
- struct timeval tv;
- int nCount;
- gettimeofday(&tv, NULL);
- nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
- return nCount;
-}
-
-#endif
-
-
-static int BMK_GetMilliSpan( int nTimeStart )
-{
- int nSpan = BMK_GetMilliStart() - nTimeStart;
- if ( nSpan < 0 )
- nSpan += 0x100000 * 1000;
- return nSpan;
-}
-
-
-static size_t BMK_findMaxMem(U64 requiredMem)
-{
- size_t step = (64 MB);
- BYTE* testmem=NULL;
-
- requiredMem = (((requiredMem >> 26) + 1) << 26);
- requiredMem += 2*step;
- if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
-
- while (!testmem)
- {
- requiredMem -= step;
- testmem = (BYTE*) malloc ((size_t)requiredMem);
- }
-
- free (testmem);
- return (size_t) (requiredMem - step);
-}
-
-
-static U64 BMK_GetFileSize(char* infilename)
-{
- int r;
-#if defined(_MSC_VER)
- struct _stat64 statbuf;
- r = _stat64(infilename, &statbuf);
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
-#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
- return (U64)statbuf.st_size;
-}
-
-
-//*********************************************************
-// Public function
-//*********************************************************
-
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
-{
- int fileIdx=0;
- char* orig_buff;
- struct compressionParameters compP;
- int cfunctionId;
-
- U64 totals = 0;
- U64 totalz = 0;
- double totalc = 0.;
- double totald = 0.;
-
-
- // Init
- if (cLevel <3) cfunctionId = 0; else cfunctionId = 1;
- switch (cfunctionId)
- {
-#ifdef COMPRESSOR0
- case 0 : compP.compressionFunction = COMPRESSOR0; break;
-#endif
-#ifdef COMPRESSOR1
- case 1 : compP.compressionFunction = COMPRESSOR1; break;
-#endif
- default : compP.compressionFunction = DEFAULTCOMPRESSOR;
- }
- compP.decompressionFunction = LZ4_decompress_fast;
-
- // Loop for each file
- while (fileIdx<nbFiles)
- {
- FILE* inFile;
- char* inFileName;
- U64 inFileSize;
- size_t benchedSize;
- int nbChunks;
- int maxCompressedChunkSize;
- size_t readSize;
- char* compressedBuffer; int compressedBuffSize;
- struct chunkParameters* chunkP;
- U32 crcOrig;
-
- // Check file existence
- inFileName = fileNamesTable[fileIdx++];
- inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
- DISPLAY( "Pb opening %s\n", inFileName);
- return 11;
- }
-
- // Memory allocation & restrictions
- inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- {
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
-
- // Alloc
- chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));
- orig_buff = (char*)malloc((size_t )benchedSize);
- nbChunks = (int) (benchedSize / chunkSize) + 1;
- maxCompressedChunkSize = LZ4_compressBound(chunkSize);
- compressedBuffSize = nbChunks * maxCompressedChunkSize;
- compressedBuffer = (char*)malloc((size_t )compressedBuffSize);
-
-
- if (!orig_buff || !compressedBuffer)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(orig_buff);
- free(compressedBuffer);
- free(chunkP);
- fclose(inFile);
- return 12;
- }
-
- // Init chunks data
- {
- int i;
- size_t remaining = benchedSize;
- char* in = orig_buff;
- char* out = compressedBuffer;
- for (i=0; i<nbChunks; i++)
- {
- chunkP[i].id = i;
- chunkP[i].origBuffer = in; in += chunkSize;
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
- chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
- chunkP[i].compressedSize = 0;
- }
- }
-
- // Fill input buffer
- DISPLAY("Loading %s... \r", inFileName);
- readSize = fread(orig_buff, 1, benchedSize, inFile);
- fclose(inFile);
-
- if (readSize != benchedSize)
- {
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
- free(orig_buff);
- free(compressedBuffer);
- free(chunkP);
- return 13;
- }
-
- // Calculating input Checksum
- crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0);
-
-
- // Bench
- {
- int loopNb, chunkNb;
- size_t cSize=0;
- double fastestC = 100000000., fastestD = 100000000.;
- double ratio=0.;
- U32 crcCheck=0;
-
- DISPLAY("\r%79s\r", "");
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)
- {
- int nbLoops;
- int milliTime;
-
- // Compression
- DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize);
- { size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } // warmimg up memory
-
- nbLoops = 0;
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
- nbLoops++;
- }
- milliTime = BMK_GetMilliSpan(milliTime);
-
- if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
- cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
- ratio = (double)cSize/(double)benchedSize*100.;
-
- DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.);
-
- // Decompression
- { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing area, for CRC checking
-
- nbLoops = 0;
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
- nbLoops++;
- }
- milliTime = BMK_GetMilliSpan(milliTime);
-
- if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
- DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
-
- // CRC Checking
- crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
- if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
- }
-
- if (crcOrig==crcCheck)
- {
- if (ratio<100.)
- DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
- else
- DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
- }
- totals += benchedSize;
- totalz += cSize;
- totalc += fastestC;
- totald += fastestD;
- }
-
- free(orig_buff);
- free(compressedBuffer);
- free(chunkP);
- }
-
- if (nbFiles > 1)
- DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
-
- if (BMK_pause) { DISPLAY("press enter...\n"); getchar(); }
-
- return 0;
-}
-
-
-
+/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + +// Unix Large Files support (>4GB) +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) || defined(_WIN32) +# define BMK_LEGACY_TIMER 1 +#endif + + +//************************************** +// Includes +//************************************** +#include <stdlib.h> // malloc +#include <stdio.h> // fprintf, fopen, ftello64 +#include <sys/types.h> // stat64 +#include <sys/stat.h> // stat64 + +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include <sys/timeb.h> // timeb, ftime +#else +# include <sys/time.h> // gettimeofday +#endif + +#include "lz4.h" +#define COMPRESSOR0 LZ4_compress +#include "lz4hc.h" +#define COMPRESSOR1 LZ4_compressHC +#define DEFAULTCOMPRESSOR COMPRESSOR0 + +#include "xxhash.h" + + +//************************************** +// Compiler specifics +//************************************** +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + +// GCC does not support _rotl outside of Windows +#if !defined(_WIN32) +# define _rotl(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +//************************************** +// Constants +//************************************** +#define NBLOOPS 3 +#define TIMELOOP 2000 + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define KNUTH 2654435761U +#define MAX_MEM (2 GB - 64 MB) +#define DEFAULT_CHUNKSIZE (4 MB) + + +//************************************** +// Local structures +//************************************** +struct chunkParameters +{ + U32 id; + char* origBuffer; + char* compressedBuffer; + int origSize; + int compressedSize; +}; + +struct compressionParameters +{ + int (*compressionFunction)(const char*, char*, int); + int (*decompressionFunction)(const char*, char*, int); +}; + + +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + + +//************************************** +// Benchmark Parameters +//************************************** +static int chunkSize = DEFAULT_CHUNKSIZE; +static int nbIterations = NBLOOPS; +static int BMK_pause = 0; + +void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } + +void BMK_SetNbIterations(int nbLoops) +{ + nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", nbIterations); +} + +void BMK_SetPause() { BMK_pause = 1; } + + +//********************************************************* +// Private functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart() +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart() +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = (64 MB); + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + while (!testmem) + { + requiredMem -= step; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +//********************************************************* +// Public function +//********************************************************* + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) +{ + int fileIdx=0; + char* orig_buff; + struct compressionParameters compP; + int cfunctionId; + + U64 totals = 0; + U64 totalz = 0; + double totalc = 0.; + double totald = 0.; + + + // Init + if (cLevel <3) cfunctionId = 0; else cfunctionId = 1; + switch (cfunctionId) + { +#ifdef COMPRESSOR0 + case 0 : compP.compressionFunction = COMPRESSOR0; break; +#endif +#ifdef COMPRESSOR1 + case 1 : compP.compressionFunction = COMPRESSOR1; break; +#endif + default : compP.compressionFunction = DEFAULTCOMPRESSOR; + } + compP.decompressionFunction = LZ4_decompress_fast; + + // Loop for each file + while (fileIdx<nbFiles) + { + FILE* inFile; + char* inFileName; + U64 inFileSize; + size_t benchedSize; + int nbChunks; + int maxCompressedChunkSize; + size_t readSize; + char* compressedBuffer; int compressedBuffSize; + struct chunkParameters* chunkP; + U32 crcOrig; + + // Check file existence + inFileName = fileNamesTable[fileIdx++]; + inFile = fopen( inFileName, "rb" ); + if (inFile==NULL) + { + DISPLAY( "Pb opening %s\n", inFileName); + return 11; + } + + // Memory allocation & restrictions + inFileSize = BMK_GetFileSize(inFileName); + benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2; + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + } + + // Alloc + chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*)malloc((size_t )benchedSize); + nbChunks = (int) (benchedSize / chunkSize) + 1; + maxCompressedChunkSize = LZ4_compressBound(chunkSize); + compressedBuffSize = nbChunks * maxCompressedChunkSize; + compressedBuffer = (char*)malloc((size_t )compressedBuffSize); + + + if (!orig_buff || !compressedBuffer) + { + DISPLAY("\nError: not enough memory!\n"); + free(orig_buff); + free(compressedBuffer); + free(chunkP); + fclose(inFile); + return 12; + } + + // Init chunks data + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressedBuffer; + for (i=0; i<nbChunks; i++) + { + chunkP[i].id = i; + chunkP[i].origBuffer = in; in += chunkSize; + if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + + // Fill input buffer + DISPLAY("Loading %s... \r", inFileName); + readSize = fread(orig_buff, 1, benchedSize, inFile); + fclose(inFile); + + if (readSize != benchedSize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(orig_buff); + free(compressedBuffer); + free(chunkP); + return 13; + } + + // Calculating input Checksum + crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0); + + + // Bench + { + int loopNb, chunkNb; + size_t cSize=0; + double fastestC = 100000000., fastestD = 100000000.; + double ratio=0.; + U32 crcCheck=0; + + DISPLAY("\r%79s\r", ""); + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + int nbLoops; + int milliTime; + + // Compression + DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize); + { size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } // warmimg up memory + + nbLoops = 0; + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + for (chunkNb=0; chunkNb<nbChunks; chunkNb++) + chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); + nbLoops++; + } + milliTime = BMK_GetMilliSpan(milliTime); + + if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops; + cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; + ratio = (double)cSize/(double)benchedSize*100.; + + DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.); + + // Decompression + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing area, for CRC checking + + nbLoops = 0; + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + for (chunkNb=0; chunkNb<nbChunks; chunkNb++) + chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize); + nbLoops++; + } + milliTime = BMK_GetMilliSpan(milliTime); + + if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops; + DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + + // CRC Checking + crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0); + if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; } + } + + if (crcOrig==crcCheck) + { + if (ratio<100.) + DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + else + DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + } + totals += benchedSize; + totalz += cSize; + totalc += fastestC; + totald += fastestD; + } + + free(orig_buff); + free(compressedBuffer); + free(chunkP); + } + + if (nbFiles > 1) + DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); + + if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); } + + return 0; +} + + + @@ -1,41 +1,41 @@ -/*
- bench.h - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2013
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);
-
-// Parameters
-void BMK_SetBlocksize(int bsize);
-void BMK_SetNbIterations(int nbLoops);
-void BMK_SetPause();
-
-
-
-#if defined (__cplusplus)
-}
-#endif
+/* + bench.h - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel); + +// Parameters +void BMK_SetBlocksize(int bsize); +void BMK_SetNbIterations(int nbLoops); +void BMK_SetPause(); + + + +#if defined (__cplusplus) +} +#endif diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index dd26bd9..b922641 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -2,12 +2,11 @@ PROJECT(LZ4 C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library")
set(CPACK_PACKAGE_VERSION_MAJOR 0)
set(CPACK_PACKAGE_VERSION_MINOR 0)
-set(CPACK_PACKAGE_VERSION_PATCH r107)
+set(CPACK_PACKAGE_VERSION_PATCH r110)
#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL)
set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")
include(CPack)
-
cmake_minimum_required (VERSION 2.6)
INCLUDE (CheckTypeSize)
check_type_size("void *" SIZEOF_VOID_P)
@@ -16,15 +15,10 @@ IF( ${SIZEOF_VOID_P} STREQUAL "8" ) MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P})
ENDIF()
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
-
-if (BUILD_SHARED_LIBS)
- message(STATUS "Enable shared library building")
-else(BUILD_SHARED_LIBS)
- message(STATUS "Disable shared library building")
-endif(BUILD_SHARED_LIBS)
+option(BUILD_TOOLS "Build the command line tools" ON)
+option(BUILD_LIBS "Build the libraries in addition to the tools" OFF)
-if(UNIX AND BUILD_SHARED_LIBS)
+if(UNIX AND BUILD_LIBS)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
add_definitions(-fPIC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
@@ -34,62 +28,42 @@ set(SRC_DIR ../) set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h )
set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c)
-if(NOT BUILD_SHARED_LIBS)
+if(BUILD_TOOLS AND NOT BUILD_LIBS)
set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})
endif()
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
- message(STATUS "Build 64bit executable binary")
- add_executable(lz4c64 ${LZ4_SRCS})
- install(TARGETS lz4c64 RUNTIME DESTINATION "bin/")
- if(NOT BUILD_SHARED_LIBS)
- message(STATUS "Build 32bit executable binary")
- add_executable(lz4c32 ${LZ4_SRCS})
- install(TARGETS lz4c32 RUNTIME DESTINATION "bin/")
-
- SET_TARGET_PROPERTIES(lz4c32 PROPERTIES
- COMPILE_FLAGS PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32")
- endif()
-else()
- message(STATUS "Build 32bit executable binary")
- add_executable(lz4c32 ${LZ4_SRCS})
- install(TARGETS lz4c32 RUNTIME DESTINATION "bin/")
+if(BUILD_TOOLS)
+ add_executable(lz4 ${LZ4_SRCS})
+ set_target_properties(lz4 PROPERTIES COMPILE_DEFINITIONS DISABLE_LZ4C_LEGACY_OPTIONS)
+ install(TARGETS lz4 RUNTIME DESTINATION "bin/")
+ add_executable(lz4c ${LZ4_SRCS})
+ install(TARGETS lz4c RUNTIME DESTINATION "bin/")
endif()
-if(BUILD_SHARED_LIBS)
- set(LZ4_SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}")
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")
- target_link_libraries(lz4c64 liblz4)
- else()
- target_link_libraries(lz4c32 liblz4)
- endif()
-
- # for static library
- add_library(liblz4_static STATIC ${LZ4_SRCS_LIB})
- set_target_properties(liblz4_static PROPERTIES OUTPUT_NAME lz4)
-
- install(TARGETS liblz4_static
- LIBRARY DESTINATION lib
- ARCHIVE DESTINATION lib
- )
-
- # for shared library o
+if(BUILD_LIBS)
add_library(liblz4 ${LZ4_SRCS_LIB})
+
set_target_properties(liblz4 PROPERTIES
- OUTPUT_NAME lz4
- SOVERSION ${LZ4_SOVERSION})
-
+ OUTPUT_NAME lz4
+ SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}"
+ )
+
install(TARGETS liblz4
- LIBRARY DESTINATION lib
- ARCHIVE DESTINATION lib
+ LIBRARY DESTINATION lib
+ ARCHIVE DESTINATION lib
)
-
+
install(FILES
- ${SRC_DIR}/lz4.h
- ${SRC_DIR}/lz4hc.h
- DESTINATION include)
-
-endif(BUILD_SHARED_LIBS)
+ ${SRC_DIR}/lz4.h
+ ${SRC_DIR}/lz4hc.h
+ DESTINATION include
+ )
+
+ if(BUILD_TOOLS)
+ target_link_libraries(lz4 liblz4)
+ target_link_libraries(lz4c liblz4)
+ endif()
+endif()
#warnings
@@ -98,8 +72,8 @@ ADD_DEFINITIONS("-Wall") ADD_DEFINITIONS("-W")
ADD_DEFINITIONS("-Wundef")
ADD_DEFINITIONS("-Wcast-align")
-ADD_DEFINITIONS("-Wno-implicit-function-declaration")
-ADD_DEFINITIONS("-Os -march=native -std=c99")
+ADD_DEFINITIONS("-std=c99")
+ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"")
INCLUDE_DIRECTORIES (${SRC_DIR})
diff --git a/fullbench.c b/fullbench.c index 88e37b6..c465c88 100644 --- a/fullbench.c +++ b/fullbench.c @@ -1,706 +1,742 @@ -/*
- bench.c - Demo program to benchmark open-source compression algorithm
- Copyright (C) Yann Collet 2012-2013
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-//**************************************
-// Compiler Options
-//**************************************
-// Disable some Visual warning messages
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE // VS2005
-
-// Unix Large Files support (>4GB)
-#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
-# define _LARGEFILE_SOURCE
-# define _FILE_OFFSET_BITS 64
-#elif ! defined(__LP64__) // No point defining Large file for 64 bit
-# define _LARGEFILE64_SOURCE
-#endif
-
-// S_ISREG & gettimeofday() are not supported by MSVC
-#if defined(_MSC_VER) || defined(_WIN32)
-# define BMK_LEGACY_TIMER 1
-#endif
-
-
-//**************************************
-// Includes
-//**************************************
-#include <stdlib.h> // malloc
-#include <stdio.h> // fprintf, fopen, ftello64
-#include <sys/types.h> // stat64
-#include <sys/stat.h> // stat64
-
-// Use ftime() if gettimeofday() is not available on your target
-#if defined(BMK_LEGACY_TIMER)
-# include <sys/timeb.h> // timeb, ftime
-#else
-# include <sys/time.h> // gettimeofday
-#endif
-
-#include "lz4.h"
-#define COMPRESSOR0 LZ4_compress
-#include "lz4hc.h"
-#define COMPRESSOR1 LZ4_compressHC
-#define DEFAULTCOMPRESSOR COMPRESSOR0
-
-#include "xxhash.h"
-
-
-//**************************************
-// Compiler Options
-//**************************************
-// S_ISREG & gettimeofday() are not supported by MSVC
-#if !defined(S_ISREG)
-# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
-#endif
-
-// GCC does not support _rotl outside of Windows
-#if !defined(_WIN32)
-# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
-#endif
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-
-//****************************
-// Constants
-//****************************
-#define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
-#ifndef LZ4_VERSION
-# define LZ4_VERSION ""
-#endif
-#define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
-
-#define NBLOOPS 6
-#define TIMELOOP 2500
-
-#define KNUTH 2654435761U
-#define MAX_MEM (1984<<20)
-#define DEFAULT_CHUNKSIZE (4<<20)
-
-#define ALL_COMPRESSORS -1
-#define ALL_DECOMPRESSORS -1
-
-
-//**************************************
-// Local structures
-//**************************************
-struct chunkParameters
-{
- U32 id;
- char* origBuffer;
- char* compressedBuffer;
- int origSize;
- int compressedSize;
-};
-
-
-//**************************************
-// MACRO
-//**************************************
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-
-
-
-//**************************************
-// Benchmark Parameters
-//**************************************
-static int chunkSize = DEFAULT_CHUNKSIZE;
-static int nbIterations = NBLOOPS;
-static int BMK_pause = 0;
-static int compressionTest = 1;
-static int decompressionTest = 1;
-static int compressionAlgo = ALL_COMPRESSORS;
-static int decompressionAlgo = ALL_DECOMPRESSORS;
-
-void BMK_SetBlocksize(int bsize)
-{
- chunkSize = bsize;
- DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
-}
-
-void BMK_SetNbIterations(int nbLoops)
-{
- nbIterations = nbLoops;
- DISPLAY("- %i iterations -\n", nbIterations);
-}
-
-void BMK_SetPause()
-{
- BMK_pause = 1;
-}
-
-//*********************************************************
-// Private functions
-//*********************************************************
-
-#if defined(BMK_LEGACY_TIMER)
-
-static int BMK_GetMilliStart()
-{
- // Based on Legacy ftime()
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)
- // Use GetMilliSpan to correct for rollover
- struct timeb tb;
- int nCount;
- ftime( &tb );
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
- return nCount;
-}
-
-#else
-
-static int BMK_GetMilliStart()
-{
- // Based on newer gettimeofday()
- // Use GetMilliSpan to correct for rollover
- struct timeval tv;
- int nCount;
- gettimeofday(&tv, NULL);
- nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
- return nCount;
-}
-
-#endif
-
-
-static int BMK_GetMilliSpan( int nTimeStart )
-{
- int nSpan = BMK_GetMilliStart() - nTimeStart;
- if ( nSpan < 0 )
- nSpan += 0x100000 * 1000;
- return nSpan;
-}
-
-
-static size_t BMK_findMaxMem(U64 requiredMem)
-{
- size_t step = (64U<<20); // 64 MB
- BYTE* testmem=NULL;
-
- requiredMem = (((requiredMem >> 25) + 1) << 26);
- if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
-
- requiredMem += 2*step;
- while (!testmem)
- {
- requiredMem -= step;
- testmem = (BYTE*) malloc ((size_t)requiredMem);
- }
-
- free (testmem);
- return (size_t) (requiredMem - step);
-}
-
-
-static U64 BMK_GetFileSize(char* infilename)
-{
- int r;
-#if defined(_MSC_VER)
- struct _stat64 statbuf;
- r = _stat64(infilename, &statbuf);
-#else
- struct stat statbuf;
- r = stat(infilename, &statbuf);
-#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
- return (U64)statbuf.st_size;
-}
-
-
-//*********************************************************
-// Public function
-//*********************************************************
-
-static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
-{
- return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
-}
-
-static void* ctx;
-static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)
-{
- return LZ4_compress_continue(ctx, in, out, inSize);
-}
-
-static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
-{
- return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
-}
-
-static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
-{
- return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
-}
-
-static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
-{
- return LZ4_compressHC_continue(ctx, in, out, inSize);
-}
-
-static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
-{
- return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
-}
-
-static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
-{
- (void)inSize;
- LZ4_decompress_fast(in, out, outSize);
- return outSize;
-}
-
-static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
-{
- (void)inSize;
- LZ4_decompress_fast_withPrefix64k(in, out, outSize);
- return outSize;
-}
-
-static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
-{
- return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
-}
-
-int fullSpeedBench(char** fileNamesTable, int nbFiles)
-{
- int fileIdx=0;
- char* orig_buff;
-# define NB_COMPRESSION_ALGORITHMS 8
-# define MINCOMPRESSIONCHAR '0'
-# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
- static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", "LZ4_compressHC", "LZ4_compressHC_limitedOutput", "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
- double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
- double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
-# define NB_DECOMPRESSION_ALGORITHMS 5
-# define MINDECOMPRESSIONCHAR '0'
-# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
- static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" };
- double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
-
- U64 totals = 0;
-
-
- // Loop for each file
- while (fileIdx<nbFiles)
- {
- FILE* inFile;
- char* inFileName;
- U64 inFileSize;
- size_t benchedSize;
- int nbChunks;
- int maxCompressedChunkSize;
- struct chunkParameters* chunkP;
- size_t readSize;
- char* compressed_buff; int compressedBuffSize;
- U32 crcOriginal;
-
- // Check file existence
- inFileName = fileNamesTable[fileIdx++];
- inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
- DISPLAY( "Pb opening %s\n", inFileName);
- return 11;
- }
-
- // Memory allocation & restrictions
- inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- {
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
-
- // Alloc
- chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));
- orig_buff = (char*) malloc((size_t)benchedSize);
- nbChunks = (int) (benchedSize / chunkSize) + 1;
- maxCompressedChunkSize = LZ4_compressBound(chunkSize);
- compressedBuffSize = nbChunks * maxCompressedChunkSize;
- compressed_buff = (char*)malloc((size_t)compressedBuffSize);
-
-
- if(!orig_buff || !compressed_buff)
- {
- DISPLAY("\nError: not enough memory!\n");
- free(orig_buff);
- free(compressed_buff);
- free(chunkP);
- fclose(inFile);
- return 12;
- }
-
- // Init chunks data
- {
- int i;
- size_t remaining = benchedSize;
- char* in = orig_buff;
- char* out = compressed_buff;
- for (i=0; i<nbChunks; i++)
- {
- chunkP[i].id = i;
- chunkP[i].origBuffer = in; in += chunkSize;
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
- chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
- chunkP[i].compressedSize = 0;
- }
- }
-
- // Fill input buffer
- DISPLAY("Loading %s... \r", inFileName);
- readSize = fread(orig_buff, 1, benchedSize, inFile);
- fclose(inFile);
-
- if(readSize != benchedSize)
- {
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
- free(orig_buff);
- free(compressed_buff);
- free(chunkP);
- return 13;
- }
-
- // Calculating input Checksum
- crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);
-
-
- // Bench
- {
- int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
- size_t cSize=0;
- double ratio=0.;
-
- DISPLAY("\r%79s\r", "");
- DISPLAY(" %s : \n", inFileName);
-
- // Compression Algorithms
- for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
- {
- char* cName = compressionNames[cAlgNb];
- int (*compressionFunction)(const char*, char*, int);
- void* (*initFunction)(const char*) = NULL;
- double bestTime = 100000000.;
-
- if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
-
- switch(cAlgNb)
- {
- case 0: compressionFunction = LZ4_compress; break;
- case 1: compressionFunction = local_LZ4_compress_limitedOutput; break;
- case 2: compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
- case 3: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
- case 4: compressionFunction = LZ4_compressHC; break;
- case 5: compressionFunction = local_LZ4_compressHC_limitedOutput; break;
- case 6: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
- case 7: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
- default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
- }
-
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)
- {
- double averageTime;
- int milliTime;
-
- DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
- { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
-
- nb_loops = 0;
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- {
- chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
- if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
- }
- if (initFunction!=NULL) free(ctx);
- nb_loops++;
- }
- milliTime = BMK_GetMilliSpan(milliTime);
-
- averageTime = (double)milliTime / nb_loops;
- if (averageTime < bestTime) bestTime = averageTime;
- cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
- ratio = (double)cSize/(double)benchedSize*100.;
- DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
- }
-
- if (ratio<100.)
- DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
- else
- DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
-
- totalCTime[cAlgNb] += bestTime;
- totalCSize[cAlgNb] += cSize;
- }
-
- // Prepare layout for decompression
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- {
- chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
- if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
- }
- { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
-
- // Decompression Algorithms
- for (dAlgNb=0; (dAlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
- {
- char* dName = decompressionNames[dAlgNb];
- int (*decompressionFunction)(const char*, char*, int, int);
- double bestTime = 100000000.;
-
- if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
-
- switch(dAlgNb)
- {
- case 0: decompressionFunction = local_LZ4_decompress_fast; break;
- case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;
- case 2: decompressionFunction = LZ4_decompress_safe; break;
- case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
- case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break;
- default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
- }
-
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)
- {
- double averageTime;
- int milliTime;
- U32 crcDecoded;
-
- DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
-
- nb_loops = 0;
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- {
- int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
- if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1);
- }
- nb_loops++;
- }
- milliTime = BMK_GetMilliSpan(milliTime);
-
- averageTime = (double)milliTime / nb_loops;
- if (averageTime < bestTime) bestTime = averageTime;
-
- DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
-
- // CRC Checking
- crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
- if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
- }
-
- DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
-
- totalDTime[dAlgNb] += bestTime;
- }
-
- totals += benchedSize;
- }
-
- free(orig_buff);
- free(compressed_buff);
- free(chunkP);
- }
-
- if (nbFiles > 1)
- {
- int AlgNb;
-
- DISPLAY(" ** TOTAL ** : \n");
- for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
- {
- char* cName = compressionNames[AlgNb];
- if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
- DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
- }
- for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
- {
- char* dName = decompressionNames[AlgNb];
- if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
- DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
- }
- }
-
- if (BMK_pause) { printf("press enter...\n"); getchar(); }
-
- return 0;
-}
-
-
-int usage(char* exename)
-{
- DISPLAY( "Usage :\n");
- DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
- DISPLAY( "Arguments :\n");
- DISPLAY( " -c : compression tests only\n");
- DISPLAY( " -d : decompression tests only\n");
- DISPLAY( " -H/-h : Help (this text + advanced options)\n");
- return 0;
-}
-
-int usage_advanced()
-{
- DISPLAY( "\nAdvanced options :\n");
- DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);
- DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
- DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
- DISPLAY( " -B# : Block size [4-7](default : 7)\n");
- //DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
- return 0;
-}
-
-int badusage(char* exename)
-{
- DISPLAY("Wrong parameters\n");
- usage(exename);
- return 0;
-}
-
-int main(int argc, char** argv)
-{
- int i,
- filenamesStart=2;
- char* exename=argv[0];
- char* input_filename=0;
-
- // Welcome message
- DISPLAY( WELCOME_MESSAGE);
-
- if (argc<2) { badusage(exename); return 1; }
-
- for(i=1; i<argc; i++)
- {
- char* argument = argv[i];
-
- if(!argument) continue; // Protection if argument empty
-
- // Decode command (note : aggregated commands are allowed)
- if (argument[0]=='-')
- {
- while (argument[1]!=0)
- {
- argument ++;
-
- switch(argument[0])
- {
- // Select compression algorithm only
- case 'c':
- decompressionTest = 0;
- if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR))
- compressionAlgo = argument[1] - '0', argument++;
- break;
-
- // Select decompression algorithm only
- case 'd':
- compressionTest = 0;
- if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR))
- decompressionAlgo = argument[1] - '0', argument++;
- break;
-
- // Display help on usage
- case 'h' :
- case 'H': usage(exename); usage_advanced(); return 0;
-
- // Modify Block Properties
- case 'B':
- while (argument[1]!=0)
- switch(argument[1])
- {
- case '4':
- case '5':
- case '6':
- case '7':
- {
- int B = argument[1] - '0';
- int S = 1 << (8 + 2*B);
- BMK_SetBlocksize(S);
- argument++;
- break;
- }
- case 'D': argument++; break;
- default : goto _exit_blockProperties;
- }
-_exit_blockProperties:
- break;
-
- // Modify Nb Iterations
- case 'i':
- if ((argument[1] >='1') && (argument[1] <='9'))
- {
- int iters = argument[1] - '0';
- BMK_SetNbIterations(iters);
- argument++;
- }
- break;
-
- // Pause at the end (hidden option)
- case 'p': BMK_SetPause(); break;
-
- // Unrecognised command
- default : badusage(exename); return 1;
- }
- }
- continue;
- }
-
- // first provided filename is input
- if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
-
- }
-
- // No input filename ==> Error
- if(!input_filename) { badusage(exename); return 1; }
-
- return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
-
-}
-
+/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + +// Unix Large Files support (>4GB) +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +# define _FILE_OFFSET_BITS 64 +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) || defined(_WIN32) +# define BMK_LEGACY_TIMER 1 +#endif + + +//************************************** +// Includes +//************************************** +#include <stdlib.h> // malloc +#include <stdio.h> // fprintf, fopen, ftello64 +#include <sys/types.h> // stat64 +#include <sys/stat.h> // stat64 + +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include <sys/timeb.h> // timeb, ftime +#else +# include <sys/time.h> // gettimeofday +#endif + +#include "lz4.h" +#define COMPRESSOR0 LZ4_compress +#include "lz4hc.h" +#define COMPRESSOR1 LZ4_compressHC +#define DEFAULTCOMPRESSOR COMPRESSOR0 + +#include "xxhash.h" + + +//************************************** +// Compiler Options +//************************************** +// S_ISREG & gettimeofday() are not supported by MSVC +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + +// GCC does not support _rotl outside of Windows +#if !defined(_WIN32) +# define _rotl(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +//**************************** +// Constants +//**************************** +#define PROGRAM_DESCRIPTION "LZ4 speed analyzer" +#ifndef LZ4_VERSION +# define LZ4_VERSION "" +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__ + +#define NBLOOPS 6 +#define TIMELOOP 2500 + +#define KNUTH 2654435761U +#define MAX_MEM (1984<<20) +#define DEFAULT_CHUNKSIZE (4<<20) + +#define ALL_COMPRESSORS -1 +#define ALL_DECOMPRESSORS -1 + + +//************************************** +// Local structures +//************************************** +struct chunkParameters +{ + U32 id; + char* origBuffer; + char* compressedBuffer; + int origSize; + int compressedSize; +}; + + +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + + +//************************************** +// Benchmark Parameters +//************************************** +static int chunkSize = DEFAULT_CHUNKSIZE; +static int nbIterations = NBLOOPS; +static int BMK_pause = 0; +static int compressionTest = 1; +static int decompressionTest = 1; +static int compressionAlgo = ALL_COMPRESSORS; +static int decompressionAlgo = ALL_DECOMPRESSORS; + +void BMK_SetBlocksize(int bsize) +{ + chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); +} + +void BMK_SetNbIterations(int nbLoops) +{ + nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", nbIterations); +} + +void BMK_SetPause() +{ + BMK_pause = 1; +} + +//********************************************************* +// Private functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart() +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart() +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = (64U<<20); // 64 MB + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 25) + 1) << 26); + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + requiredMem += 2*step; + while (!testmem) + { + requiredMem -= step; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +//********************************************************* +// Public function +//********************************************************* + +static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* stateLZ4; +static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize) +{ + return LZ4_compress_withState(stateLZ4, in, out, inSize); +} + +static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* ctx; +static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize) +{ + return LZ4_compress_continue(ctx, in, out, inSize); +} + +static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* stateLZ4HC; +static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compress_withState(stateLZ4HC, in, out, inSize); +} + +static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_continue(ctx, in, out, inSize); +} + +static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + LZ4_decompress_fast(in, out, outSize); + return outSize; +} + +static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + LZ4_decompress_fast_withPrefix64k(in, out, outSize); + return outSize; +} + +static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) +{ + return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); +} + +int fullSpeedBench(char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; + char* orig_buff; +# define NB_COMPRESSION_ALGORITHMS 12 +# define MINCOMPRESSIONCHAR '0' +# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) + static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", + "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState", + "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", + "LZ4_compressHC", "LZ4_compressHC_limitedOutput", + "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC", + "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" }; + double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0}; + double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0}; +# define NB_DECOMPRESSION_ALGORITHMS 5 +# define MINDECOMPRESSIONCHAR '0' +# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS) + static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" }; + double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0}; + + U64 totals = 0; + + + // Loop for each file + while (fileIdx<nbFiles) + { + FILE* inFile; + char* inFileName; + U64 inFileSize; + size_t benchedSize; + int nbChunks; + int maxCompressedChunkSize; + struct chunkParameters* chunkP; + size_t readSize; + char* compressed_buff; int compressedBuffSize; + U32 crcOriginal; + + + // Init + stateLZ4 = malloc(LZ4_sizeofState()); + stateLZ4HC = malloc(LZ4_sizeofStateHC()); + + // Check file existence + inFileName = fileNamesTable[fileIdx++]; + inFile = fopen( inFileName, "rb" ); + if (inFile==NULL) + { + DISPLAY( "Pb opening %s\n", inFileName); + return 11; + } + + // Memory allocation & restrictions + inFileSize = BMK_GetFileSize(inFileName); + benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2; + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + } + + // Alloc + chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*) malloc((size_t)benchedSize); + nbChunks = (int) (benchedSize / chunkSize) + 1; + maxCompressedChunkSize = LZ4_compressBound(chunkSize); + compressedBuffSize = nbChunks * maxCompressedChunkSize; + compressed_buff = (char*)malloc((size_t)compressedBuffSize); + + + if(!orig_buff || !compressed_buff) + { + DISPLAY("\nError: not enough memory!\n"); + free(orig_buff); + free(compressed_buff); + free(chunkP); + fclose(inFile); + return 12; + } + + // Init chunks data + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + for (i=0; i<nbChunks; i++) + { + chunkP[i].id = i; + chunkP[i].origBuffer = in; in += chunkSize; + if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + + // Fill input buffer + DISPLAY("Loading %s... \r", inFileName); + readSize = fread(orig_buff, 1, benchedSize, inFile); + fclose(inFile); + + if(readSize != benchedSize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return 13; + } + + // Calculating input Checksum + crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); + + + // Bench + { + int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; + size_t cSize=0; + double ratio=0.; + + DISPLAY("\r%79s\r", ""); + DISPLAY(" %s : \n", inFileName); + + // Compression Algorithms + for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) + { + char* cName = compressionNames[cAlgNb]; + int (*compressionFunction)(const char*, char*, int); + void* (*initFunction)(const char*) = NULL; + double bestTime = 100000000.; + + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; + + switch(cAlgNb) + { + case 0 : compressionFunction = LZ4_compress; break; + case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break; + case 2 : compressionFunction = local_LZ4_compress_withState; break; + case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break; + case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break; + case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break; + case 6 : compressionFunction = LZ4_compressHC; break; + case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break; + case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break; + case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break; + case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break; + case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break; + default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; + } + + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + double averageTime; + int milliTime; + + DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); + { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory + + nb_loops = 0; + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer); + for (chunkNb=0; chunkNb<nbChunks; chunkNb++) + { + chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); + if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1); + } + if (initFunction!=NULL) free(ctx); + nb_loops++; + } + milliTime = BMK_GetMilliSpan(milliTime); + + averageTime = (double)milliTime / nb_loops; + if (averageTime < bestTime) bestTime = averageTime; + cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; + ratio = (double)cSize/(double)benchedSize*100.; + DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + } + + if (ratio<100.) + DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + else + DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + + totalCTime[cAlgNb] += bestTime; + totalCSize[cAlgNb] += cSize; + } + + // Prepare layout for decompression + for (chunkNb=0; chunkNb<nbChunks; chunkNb++) + { + chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); + if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1); + } + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking + + // Decompression Algorithms + for (dAlgNb=0; (dAlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++) + { + char* dName = decompressionNames[dAlgNb]; + int (*decompressionFunction)(const char*, char*, int, int); + double bestTime = 100000000.; + + if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue; + + switch(dAlgNb) + { + case 0: decompressionFunction = local_LZ4_decompress_fast; break; + case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break; + case 2: decompressionFunction = LZ4_decompress_safe; break; + case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break; + case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break; + default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; + } + + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + double averageTime; + int milliTime; + U32 crcDecoded; + + DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize); + + nb_loops = 0; + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + for (chunkNb=0; chunkNb<nbChunks; chunkNb++) + { + int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize); + if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1); + } + nb_loops++; + } + milliTime = BMK_GetMilliSpan(milliTime); + + averageTime = (double)milliTime / nb_loops; + if (averageTime < bestTime) bestTime = averageTime; + + DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + + // CRC Checking + crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); + if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } + } + + DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + + totalDTime[dAlgNb] += bestTime; + } + + totals += benchedSize; + } + + free(orig_buff); + free(compressed_buff); + free(chunkP); + } + + if (nbFiles > 1) + { + int AlgNb; + + DISPLAY(" ** TOTAL ** : \n"); + for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++) + { + char* cName = compressionNames[AlgNb]; + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue; + DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.); + } + for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++) + { + char* dName = decompressionNames[AlgNb]; + if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue; + DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.); + } + } + + if (BMK_pause) { printf("press enter...\n"); getchar(); } + + return 0; +} + + +int usage(char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c : compression tests only\n"); + DISPLAY( " -d : decompression tests only\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +int usage_advanced() +{ + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR); + DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); + DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); + DISPLAY( " -B# : Block size [4-7](default : 7)\n"); + //DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + return 0; +} + +int badusage(char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 0; +} + +int main(int argc, char** argv) +{ + int i, + filenamesStart=2; + char* exename=argv[0]; + char* input_filename=0; + + // Welcome message + DISPLAY( WELCOME_MESSAGE); + + if (argc<2) { badusage(exename); return 1; } + + for(i=1; i<argc; i++) + { + char* argument = argv[i]; + + if(!argument) continue; // Protection if argument empty + + // Decode command (note : aggregated commands are allowed) + if (argument[0]=='-') + { + while (argument[1]!=0) + { + argument ++; + + switch(argument[0]) + { + // Select compression algorithm only + case 'c': + decompressionTest = 0; + if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR)) + compressionAlgo = argument[1] - '0', argument++; + break; + + // Select decompression algorithm only + case 'd': + compressionTest = 0; + if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR)) + decompressionAlgo = argument[1] - '0', argument++; + break; + + // Display help on usage + case 'h' : + case 'H': usage(exename); usage_advanced(); return 0; + + // Modify Block Properties + case 'B': + while (argument[1]!=0) + switch(argument[1]) + { + case '4': + case '5': + case '6': + case '7': + { + int B = argument[1] - '0'; + int S = 1 << (8 + 2*B); + BMK_SetBlocksize(S); + argument++; + break; + } + case 'D': argument++; break; + default : goto _exit_blockProperties; + } +_exit_blockProperties: + break; + + // Modify Nb Iterations + case 'i': + if ((argument[1] >='1') && (argument[1] <='9')) + { + int iters = argument[1] - '0'; + BMK_SetNbIterations(iters); + argument++; + } + break; + + // Pause at the end (hidden option) + case 'p': BMK_SetPause(); break; + + // Unrecognised command + default : badusage(exename); return 1; + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + } + + // No input filename ==> Error + if(!input_filename) { badusage(exename); return 1; } + + return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); + +} + @@ -1,280 +1,303 @@ -/*
- fuzzer.c - Fuzzer test tool for LZ4
- Copyright (C) Yann Collet - Andrew Mahone 2012-2013
- Code started by Andrew Mahone, modified by Yann Collet
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-//**************************************
-// Remove Visual warning messages
-//**************************************
-#define _CRT_SECURE_NO_WARNINGS // fgets
-
-
-//**************************************
-// Includes
-//**************************************
-#include <stdlib.h>
-#include <stdio.h> // fgets, sscanf
-#include <sys/timeb.h> // timeb
-#include "lz4.h"
-#include "lz4hc.h"
-
-
-//**************************************
-// Constants
-//**************************************
-#ifndef LZ4_VERSION
-# define LZ4_VERSION ""
-#endif
-
-#define NB_ATTEMPTS (1<<17)
-#define LEN ((1<<15))
-#define SEQ_POW 2
-#define NUM_SEQ (1 << SEQ_POW)
-#define SEQ_MSK ((NUM_SEQ) - 1)
-#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0)
-#define NEW_SEQ(x) ((((x) >> 10) %10) == 0)
-#define PAGE_SIZE 4096
-#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))
-#define PRIME1 2654435761U
-#define PRIME2 2246822519U
-#define PRIME3 3266489917U
-
-
-//*********************************************************
-// Functions
-//*********************************************************
-static int FUZ_GetMilliStart()
-{
- struct timeb tb;
- int nCount;
- ftime( &tb );
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
- return nCount;
-}
-
-
-static int FUZ_GetMilliSpan( int nTimeStart )
-{
- int nSpan = FUZ_GetMilliStart() - nTimeStart;
- if ( nSpan < 0 )
- nSpan += 0x100000 * 1000;
- return nSpan;
-}
-
-
-unsigned int FUZ_rand(unsigned int* src)
-{
- *src = ((*src) * PRIME1) + PRIME2;
- return *src;
-}
-
-
-int test_canary(unsigned char *buf)
-{
- int i;
- for (i = 0; i < 2048; i++)
- if (buf[i] != buf[i + 2048])
- return 0;
- return 1;
-}
-
-
-int FUZ_SecurityTest()
-{
- char* output;
- char* input;
- int i, r;
-
- printf("Overflow test (issue 52)...\n");
- input = (char*) malloc (20<<20);
- output = (char*) malloc (20<<20);
- input[0] = 0x0F;
- input[1] = 0x00;
- input[2] = 0x00;
- for(i = 3; i < 16840000; i++)
- input[i] = 0xff;
- r = LZ4_decompress_fast(input, output, 20<<20);
-
- free(input);
- free(output);
- printf(" Passed (return = %i < 0)\n",r);
- return 0;
-}
-
-
-//int main(int argc, char *argv[]) {
-int main() {
- unsigned long long bytes = 0;
- unsigned long long cbytes = 0;
- unsigned long long hcbytes = 0;
- unsigned char buf[LEN];
- unsigned char testOut[LEN+1];
-# define FUZ_max LZ4_COMPRESSBOUND(LEN)
-# define FUZ_avail ROUND_PAGE(FUZ_max)
- const int off_full = FUZ_avail - FUZ_max;
- unsigned char cbuf[FUZ_avail + PAGE_SIZE];
- unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();
- int i, j, k, ret, len, lenHC, attemptNb;
- char userInput[30] = {0};
-# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
-# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb);
-
- printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
- printf("Select an Initialisation number (default : random) : ");
- fflush(stdout);
- if ( fgets(userInput, sizeof userInput, stdin) )
- {
- if ( sscanf(userInput, "%d", &seed) == 1 ) {}
- else seed = FUZ_GetMilliSpan(timestamp);
- }
- printf("Seed = %u\n", seed);
- randState = seed;
-
- //FUZ_SecurityTest();
-
- for (i = 0; i < 2048; i++)
- cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16;
-
- for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++)
- {
- int testNb = 0;
-
- printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
-
- for (j = 0; j < NUM_SEQ; j++) {
- seeds[j] = FUZ_rand(&randState) << 8;
- seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535;
- }
- for (j = 0; j < LEN; j++) {
- k = FUZ_rand(&randState);
- if (j == 0 || NEW_SEQ(k))
- cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK];
- if (MOD_SEQ(k)) {
- k = (FUZ_rand(&randState) >> 16) & SEQ_MSK;
- seeds[k] = FUZ_rand(&randState) << 8;
- seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535;
- }
- buf[j] = FUZ_rand(&cur_seq) >> 16;
- }
-
- // Test compression HC
- FUZ_DISPLAYTEST; // 1
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
- FUZ_CHECKTEST(ret==0, "HC compression failed despite sufficient space");
- lenHC = ret;
-
- // Test compression
- FUZ_DISPLAYTEST; // 2
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
- FUZ_CHECKTEST(ret==0, "compression failed despite sufficient space");
- len = ret;
-
- // Test decoding with output size being exactly what's necessary => must work
- FUZ_DISPLAYTEST; // 3
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
-
- // Test decoding with one byte missing => must fail
- FUZ_DISPLAYTEST; // 4
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
-
- // Test decoding with one byte too much => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
-
- // Test decoding with enough output size => must work
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
-
- // Test decoding with output size being exactly what's necessary => must work
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
-
- // Test decoding with output size being one byte too short => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
-
- // Test decoding with input size being one byte too short => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short");
-
- // Test decoding with input size being one byte too large => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
- //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
-
- // Test partial decoding with target output size being max/2 => must work
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
-
- // Test partial decoding with target output size being just below max => must work
- FUZ_DISPLAYTEST;
- ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
-
- // Test compression with output size being exactly what's necessary (should work)
- FUZ_DISPLAYTEST;
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);
- FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
- FUZ_CHECKTEST(ret==0, "compression failed despite sufficient space");
-
- // Test HC compression with output size being exactly what's necessary (should work)
- FUZ_DISPLAYTEST;
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
- FUZ_CHECKTEST(ret==0, "HC compression failed despite sufficient space");
-
- // Test compression with just one missing byte into output buffer => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);
- FUZ_CHECKTEST(ret, "compression overran output buffer");
- FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
-
- // Test HC compression with just one missing byte into output buffer => must fail
- FUZ_DISPLAYTEST;
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1);
- FUZ_CHECKTEST(ret, "HC compression overran output buffer");
-
- bytes += LEN;
- cbytes += len;
- hcbytes += lenHC;
- FUZ_rand(&randState);
- }
-
- printf("all tests completed successfully \n");
- printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
- printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
- getchar();
- return 0;
-
-_output_error:
- getchar();
- return 1;
-}
+/* + fuzzer.c - Fuzzer test tool for LZ4 + Copyright (C) Yann Collet - Andrew Mahone 2012-2013 + Code started by Andrew Mahone, modified by Yann Collet + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Remove Visual warning messages +//************************************** +#define _CRT_SECURE_NO_WARNINGS // fgets + + +//************************************** +// Includes +//************************************** +#include <stdlib.h> +#include <stdio.h> // fgets, sscanf +#include <sys/timeb.h> // timeb +#include "lz4.h" +#include "lz4hc.h" + + +//************************************** +// Constants +//************************************** +#ifndef LZ4_VERSION +# define LZ4_VERSION "" +#endif + +#define NB_ATTEMPTS (1<<17) +#define LEN ((1<<15)) +#define SEQ_POW 2 +#define NUM_SEQ (1 << SEQ_POW) +#define SEQ_MSK ((NUM_SEQ) - 1) +#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) +#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) +#define PAGE_SIZE 4096 +#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) +#define PRIME1 2654435761U +#define PRIME2 2246822519U +#define PRIME3 3266489917U + + +//********************************************************* +// Functions +//********************************************************* +static int FUZ_GetMilliStart() +{ + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + + +static int FUZ_GetMilliSpan( int nTimeStart ) +{ + int nSpan = FUZ_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +unsigned int FUZ_rand(unsigned int* src) +{ + *src = ((*src) * PRIME1) + PRIME2; + return *src; +} + + +int test_canary(unsigned char *buf) +{ + int i; + for (i = 0; i < 2048; i++) + if (buf[i] != buf[i + 2048]) + return 0; + return 1; +} + + +int FUZ_SecurityTest() +{ + char* output; + char* input; + int i, r; + + printf("Overflow test (issue 52)...\n"); + input = (char*) malloc (20<<20); + output = (char*) malloc (20<<20); + input[0] = 0x0F; + input[1] = 0x00; + input[2] = 0x00; + for(i = 3; i < 16840000; i++) + input[i] = 0xff; + r = LZ4_decompress_fast(input, output, 20<<20); + + free(input); + free(output); + printf(" Passed (return = %i < 0)\n",r); + return 0; +} + + +//int main(int argc, char *argv[]) { +int main() { + unsigned long long bytes = 0; + unsigned long long cbytes = 0; + unsigned long long hcbytes = 0; + unsigned char buf[LEN]; + unsigned char testOut[LEN+1]; +# define FUZ_max LZ4_COMPRESSBOUND(LEN) +# define FUZ_avail ROUND_PAGE(FUZ_max) + const int off_full = FUZ_avail - FUZ_max; + unsigned char cbuf[FUZ_avail + PAGE_SIZE]; + unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + int i, j, k, ret, len, lenHC, attemptNb; + char userInput[30] = {0}; +# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; } +# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb); + void* stateLZ4 = malloc(LZ4_sizeofState()); + void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); + + printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION); + printf("Select an Initialisation number (default : random) : "); + fflush(stdout); + if ( fgets(userInput, sizeof userInput, stdin) ) + { + if ( sscanf(userInput, "%d", &seed) == 1 ) {} + else seed = FUZ_GetMilliSpan(timestamp); + } + printf("Seed = %u\n", seed); + randState = seed; + + //FUZ_SecurityTest(); + + for (i = 0; i < 2048; i++) + cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16; + + for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++) + { + int testNb = 0; + + printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS); + + for (j = 0; j < NUM_SEQ; j++) { + seeds[j] = FUZ_rand(&randState) << 8; + seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535; + } + for (j = 0; j < LEN; j++) { + k = FUZ_rand(&randState); + if (j == 0 || NEW_SEQ(k)) + cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK]; + if (MOD_SEQ(k)) { + k = (FUZ_rand(&randState) >> 16) & SEQ_MSK; + seeds[k] = FUZ_rand(&randState) << 8; + seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535; + } + buf[j] = FUZ_rand(&cur_seq) >> 16; + } + + // Test compression HC + FUZ_DISPLAYTEST; // 1 + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + lenHC = ret; + + // Test compression HC using external state + FUZ_DISPLAYTEST; // 1 + ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); + + // Test compression using external state + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN); + FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); + + // Test compression + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + len = ret; + + // Test decoding with output size being exactly what's necessary => must work + FUZ_DISPLAYTEST; // 3 + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); + + // Test decoding with one byte missing => must fail + FUZ_DISPLAYTEST; // 4 + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); + + // Test decoding with one byte too much => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); + + // Test decoding with enough output size => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); + + // Test decoding with output size being exactly what's necessary => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); + + // Test decoding with output size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); + + // Test decoding with input size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short"); + + // Test decoding with input size being one byte too large => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); + //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test partial decoding with target output size being max/2 => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + // Test partial decoding with target output size being just below max => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + // Test compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test compression with output size being exactly what's necessary and external state (should work) + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); + + // Test compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); + FUZ_CHECKTEST(ret, "compression overran output buffer"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test HC compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1); + FUZ_CHECKTEST(ret, "HC compression overran output buffer"); + + bytes += LEN; + cbytes += len; + hcbytes += lenHC; + FUZ_rand(&randState); + } + + printf("all tests completed successfully \n"); + printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); + getchar(); + return 0; + +_output_error: + getchar(); + return 1; +} @@ -1,84 +1,87 @@ -\"
-\" lz4.1: This is a manual page for 'lz4' program. This file is part of the
-\" lz4 <https://code.google.com/p/lz4/> project.
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH lz4 man
-.SH NAME
-\fBlz4\fR - Extremely fast compression algorithm
-
-.SH SYNOPSIS
-.TP 5
-\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
-
-.SH DESCRIPTION
-.PP
-\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on
-the \fBLZ77\fR family of compression scheme. At the compression speed of 400
-MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features
-an extremely fast decoder, with speed in multiple GB/s per core, typically
-reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports
-following options
-
-.SH OPTIONS
-.TP
-.B \-1
- fast compression (default)
-.TP
-.B \-9
- high compression
-.TP
-.B \-d
- decompression
-.TP
-.B \-f
- overwrite output without prompting
-.TP
-.B \-h/\-H
- display help/long help and exit
-.TP
-.B \-V
- display Version number and exit
-.TP
-.B \-v
- verbose mode
-.TP
-.B \-q
- suppress warnings; specify twice to suppress errors too
-.TP
-.B \-c
- force write to standard output, even if it is the console
-.TP
-.B \-t
- test compressed file integrity
-.TP
-.B \-z
- force compression
-.TP
-.B \-B#
- block size [4-7](default : 7)
-.TP
-.B \-BD
- block dependency (improve compression ratio)
-.TP
-.B \-BX
- enable block checksum (default:disabled)
-.TP
-.B \-Sx
- disable stream checksum (default:enabled)
-.TP
-.B \-b
- benchmark file(s)
-.TP
-.B \-i#
- iteration loops [1-9](default : 3), benchmark mode only
-
-.SH BUGS
-Report bugs at:- https://code.google.com/p/lz4/
-
-.SH AUTHOR
+\" +\" lz4.1: This is a manual page for 'lz4' program. This file is part of the +\" lz4 <https://code.google.com/p/lz4/> project. +\" + +\" No hyphenation +.hy 0 +.nr HY 0 + +.TH lz4 man +.SH NAME +\fBlz4\fR - Extremely fast compression algorithm + +.SH SYNOPSIS +.TP 5 +\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE> + +.SH DESCRIPTION +.PP +\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on +the \fBLZ77\fR family of compression scheme. At the compression speed of 400 +MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features +an extremely fast decoder, with speed in multiple GB/s per core, typically +reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports +following options + +.SH OPTIONS +.TP +.B \-1 + fast compression (default) +.TP +.B \-9 + high compression +.TP +.B \-d + decompression +.TP +.B \-f + overwrite output without prompting +.TP +.B \-h/\-H + display help/long help and exit +.TP +.B \-V + display Version number and exit +.TP +.B \-v + verbose mode +.TP +.B \-q + suppress warnings; specify twice to suppress errors too +.TP +.B \-c + force write to standard output, even if it is the console +.TP +.B \-t + test compressed file integrity +.TP +.B \-z + force compression +.TP +.B \-l + use Legacy format (useful for Linux Kernel compression) +.TP +.B \-B# + block size [4-7](default : 7) +.TP +.B \-BD + block dependency (improve compression ratio) +.TP +.B \-BX + enable block checksum (default:disabled) +.TP +.B \-Sx + disable stream checksum (default:enabled) +.TP +.B \-b + benchmark file(s) +.TP +.B \-i# + iteration loops [1-9](default : 3), benchmark mode only + +.SH BUGS +Report bugs at:- https://code.google.com/p/lz4/ + +.SH AUTHOR Yann Collet
\ No newline at end of file @@ -1,822 +1,865 @@ -/*
- LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-//**************************************
-// Tuning parameters
-//**************************************
-// MEMORY_USAGE :
-// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-// Increasing memory usage improves compression ratio
-// Reduced memory usage can improve speed, due to cache effect
-// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
-#define MEMORY_USAGE 14
-
-// HEAPMODE :
-// Select how default compression functions will allocate memory for their hash table,
-// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
-#define HEAPMODE 0
-
-
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
-#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
-# define LZ4_ARCH64 1
-#else
-# define LZ4_ARCH64 0
-#endif
-
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
-#if defined (__GLIBC__)
-# include <endian.h>
-# if (__BYTE_ORDER == __BIG_ENDIAN)
-# define LZ4_BIG_ENDIAN 1
-# endif
-#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))
-# define LZ4_BIG_ENDIAN 1
-#elif defined(__sparc) || defined(__sparc__) \
- || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
- || defined(__hpux) || defined(__hppa) \
- || defined(_MIPSEB) || defined(__s390__)
-# define LZ4_BIG_ENDIAN 1
-#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
-#endif
-
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
-#if defined(__ARM_FEATURE_UNALIGNED)
-# define LZ4_FORCE_UNALIGNED_ACCESS 1
-#endif
-
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
-# define LZ4_FORCE_SW_BITCOUNT
-#endif
-
-// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
-// This option may provide a small boost to performance for some big endian cpu, although probably modest.
-// You may set this option to 1 if data will remain within closed environment.
-// This option is useless on Little_Endian CPU (such as x86)
-//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1
-
-
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99
-/* "restrict" is a known keyword */
-#else
-# define restrict // Disable restrict
-#endif
-
-#ifdef _MSC_VER // Visual Studio
-# define FORCE_INLINE static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
-# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-#endif
-
-#ifdef _MSC_VER
-# define lz4_bswap16(x) _byteswap_ushort(x)
-#else
-# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
-#endif
-
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
-# define expect(expr,value) (__builtin_expect ((expr),(value)) )
-#else
-# define expect(expr,value) (expr)
-#endif
-
-#define likely(expr) expect((expr) != 0, 1)
-#define unlikely(expr) expect((expr) != 0, 0)
-
-
-//**************************************
-// Memory routines
-//**************************************
-#include <stdlib.h> // malloc, calloc, free
-#define ALLOCATOR(n,s) calloc(n,s)
-#define FREEMEM free
-#include <string.h> // memset, memcpy
-#define MEM_INIT memset
-
-
-//**************************************
-// Includes
-//**************************************
-#include "lz4.h"
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
-# define _PACKED __attribute__ ((packed))
-#else
-# define _PACKED
-#endif
-
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-# pragma pack(1)
-# else
-# pragma pack(push, 1)
-# endif
-#endif
-
-typedef struct { U16 v; } _PACKED U16_S;
-typedef struct { U32 v; } _PACKED U32_S;
-typedef struct { U64 v; } _PACKED U64_S;
-typedef struct {size_t v;} _PACKED size_t_S;
-
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-# pragma pack(0)
-# else
-# pragma pack(pop)
-# endif
-#endif
-
-#define A16(x) (((U16_S *)(x))->v)
-#define A32(x) (((U32_S *)(x))->v)
-#define A64(x) (((U64_S *)(x))->v)
-#define AARCH(x) (((size_t_S *)(x))->v)
-
-
-//**************************************
-// Constants
-//**************************************
-#define LZ4_HASHLOG (MEMORY_USAGE-2)
-#define HASHTABLESIZE (1 << MEMORY_USAGE)
-#define HASHNBCELLS4 (1 << LZ4_HASHLOG)
-
-#define MINMATCH 4
-
-#define COPYLENGTH 8
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH+MINMATCH)
-const int LZ4_minLength = (MFLIMIT+1);
-
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))
-#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data
-
-#define MAXD_LOG 16
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
-
-#define ML_BITS 4
-#define ML_MASK ((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-
-//**************************************
-// Structures and local types
-//**************************************
-
-typedef struct {
- U32 hashTable[HASHNBCELLS4];
- const BYTE* bufferStart;
- const BYTE* base;
- const BYTE* nextBlock;
-} LZ4_Data_Structure;
-
-typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;
-typedef enum { byPtr, byU32, byU16 } tableType_t;
-
-typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive;
-
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
-typedef enum { full = 0, partial = 1 } earlyEnd_directive;
-
-
-//**************************************
-// Architecture-specific macros
-//**************************************
-#define STEPSIZE sizeof(size_t)
-#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }
-#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); }
-#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); }
-
-#if LZ4_ARCH64 // 64-bit
-# define HTYPE U32
-# define INITBASE(base) const BYTE* const base = ip
-#else // 32-bit
-# define HTYPE const BYTE*
-# define INITBASE(base) const int base = 0
-#endif
-
-#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
-# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
-# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
-#endif
-
-
-//**************************************
-// Macros
-//**************************************
-#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e;
-
-
-//****************************
-// Private functions
-//****************************
-#if LZ4_ARCH64
-
-FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
-{
-# if defined(LZ4_BIG_ENDIAN)
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
-# else
- int r;
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
-# endif
-# else
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanForward64( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll(val) >> 3);
-# else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-# endif
-# endif
-}
-
-#else
-
-FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
-{
-# if defined(LZ4_BIG_ENDIAN)
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz(val) >> 3);
-# else
- int r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
-# endif
-# else
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r;
- _BitScanForward( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz(val) >> 3);
-# else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
-# endif
-}
-
-#endif
-
-
-//****************************
-// Compression functions
-//****************************
-FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
-{
- if (tableType == byU16)
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
- else
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
-}
-
-FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
-
-FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- switch (tableType)
- {
- case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; }
- case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; }
- case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; }
- }
-}
-
-FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- U32 h = LZ4_hashPosition(p, tableType);
- LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
-}
-
-FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
- if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
- { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return
-}
-
-FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- U32 h = LZ4_hashPosition(p, tableType);
- return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
-}
-
-
-FORCE_INLINE int LZ4_compress_generic(
- void* ctx,
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize,
-
- limitedOutput_directive limitedOutput,
- tableType_t tableType,
- prefix64k_directive prefix)
-{
- const BYTE* ip = (const BYTE*) source;
- const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;
- const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);
- const BYTE* anchor = (const BYTE*) source;
- const BYTE* const iend = ip + inputSize;
- const BYTE* const mflimit = iend - MFLIMIT;
- const BYTE* const matchlimit = iend - LASTLITERALS;
-
- BYTE* op = (BYTE*) dest;
- BYTE* const oend = op + maxOutputSize;
-
- int length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
- // Init conditions
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative)
- if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block
- if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit
- if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit)
- if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals)
-
- // First Byte
- LZ4_putPosition(ip, ctx, tableType, base);
- ip++; forwardH = LZ4_hashPosition(ip, tableType);
-
- // Main Loop
- for ( ; ; )
- {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE* forwardIp = ip;
- const BYTE* ref;
- BYTE* token;
-
- // Find a match
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if unlikely(forwardIp > mflimit) { goto _last_literals; }
-
- forwardH = LZ4_hashPosition(forwardIp, tableType);
- ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
-
- } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));
-
- // Catch up
- while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; }
-
- // Encode Literal length
- length = (int)(ip - anchor);
- token = op++;
- if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit
- if (length>=(int)RUN_MASK)
- {
- int len = length-RUN_MASK;
- *token=(RUN_MASK<<ML_BITS);
- for(; len >= 255 ; len-=255) *op++ = 255;
- *op++ = (BYTE)len;
- }
- else *token = (BYTE)(length<<ML_BITS);
-
- // Copy Literals
- { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }
-
-_next_match:
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
-
- // Start Counting
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified
- anchor = ip;
- while likely(ip<matchlimit-(STEPSIZE-1))
- {
- size_t diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
- if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
- if ((ip<matchlimit) && (*ref == *ip)) ip++;
-_endCount:
-
- // Encode MatchLength
- length = (int)(ip - anchor);
- if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit
- if (length>=(int)ML_MASK)
- {
- *token += ML_MASK;
- length -= ML_MASK;
- for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
- if (length >= 255) { length-=255; *op++ = 255; }
- *op++ = (BYTE)length;
- }
- else *token += (BYTE)(length);
-
- // Test end of chunk
- if (ip > mflimit) { anchor = ip; break; }
-
- // Fill table
- LZ4_putPosition(ip-2, ctx, tableType, base);
-
- // Test next position
- ref = LZ4_getPosition(ip, ctx, tableType, base);
- LZ4_putPosition(ip, ctx, tableType, base);
- if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
-
- // Prepare next loop
- anchor = ip++;
- forwardH = LZ4_hashPosition(ip, tableType);
- }
-
-_last_literals:
- // Encode Last Literals
- {
- int lastRun = (int)(iend - anchor);
- if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (BYTE)(lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-int LZ4_compress(const char* source, char* dest, int inputSize)
-{
-#if (HEAPMODE)
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
-#else
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
-#endif
- int result;
-
- if (inputSize < (int)LZ4_64KLIMIT)
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noPrefix);
- else
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
-
-#if (HEAPMODE)
- FREEMEM(ctx);
-#endif
- return result;
-}
-
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize)
-{
- return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix);
-}
-
-
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
-{
-#if (HEAPMODE)
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
-#else
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
-#endif
- int result;
-
- if (inputSize < (int)LZ4_64KLIMIT)
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix);
- else
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
-
-#if (HEAPMODE)
- FREEMEM(ctx);
-#endif
- return result;
-}
-
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix);
-}
-
-
-//****************************
-// Stream functions
-//****************************
-
-FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
-{
- MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));
- lz4ds->bufferStart = base;
- lz4ds->base = base;
- lz4ds->nextBlock = base;
-}
-
-
-void* LZ4_create (const char* inputBuffer)
-{
- void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));
- LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);
- return lz4ds;
-}
-
-
-int LZ4_free (void* LZ4_Data)
-{
- FREEMEM(LZ4_Data);
- return (0);
-}
-
-
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
- LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;
- size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);
-
- if ( (lz4ds->base - delta > lz4ds->base) // underflow control
- || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit
- {
- size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;
- int nH;
-
- for (nH=0; nH < HASHNBCELLS4; nH++)
- {
- if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0;
- else lz4ds->hashTable[nH] -= (U32)deltaLimit;
- }
- memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
- lz4ds->base = lz4ds->bufferStart;
- lz4ds->nextBlock = lz4ds->base + 64 KB;
- }
- else
- {
- memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
- lz4ds->nextBlock -= delta;
- lz4ds->base -= delta;
- }
-
- return (char*)(lz4ds->nextBlock);
-}
-
-
-//****************************
-// Decompression functions
-//****************************
-
-// This generic decompression function cover all use cases.
-// It shall be instanciated several times, using different sets of directives
-// Note that it is essential this generic function is really inlined,
-// in order to remove useless branches during compilation optimisation.
-FORCE_INLINE int LZ4_decompress_generic(
- const char* source,
- char* dest,
- int inputSize, //
- int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer.
-
- int endOnInput, // endOnOutputSize, endOnInputSize
- int prefix64k, // noPrefix, withPrefix
- int partialDecoding, // full, partial
- int targetOutputSize // only used if partialDecoding==partial
- )
-{
- // Local Variables
- const BYTE* restrict ip = (const BYTE*) source;
- const BYTE* ref;
- const BYTE* const iend = ip + inputSize;
-
- BYTE* op = (BYTE*) dest;
- BYTE* const oend = op + outputSize;
- BYTE* cpy;
- BYTE* oexit = op + targetOutputSize;
-
- const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64
- static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
-
-
- // Special cases
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything
- if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer
- if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1);
-
-
- // Main Loop
- while (1)
- {
- unsigned token;
- size_t length;
-
- // get runlength
- token = *ip++;
- if ((length=(token>>ML_BITS)) == RUN_MASK)
- {
- unsigned s=255;
- while (((endOnInput)?ip<iend:1) && (s==255))
- {
- s = *ip++;
- length += s;
- }
- }
-
- // copy literals
- cpy = op+length;
- if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-COPYLENGTH)))
- {
- if (partialDecoding)
- {
- if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer
- if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer
- }
- else
- {
- if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed
- }
- memcpy(op, ip, length);
- ip += length;
- op += length;
- break; // Necessarily EOF, due to parsing restrictions
- }
- LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
-
- // get offset
- LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer
-
- // get matchlength
- if ((length=(token&ML_MASK)) == ML_MASK)
- {
- while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token
- {
- unsigned s = *ip++;
- length += s;
- if (s==255) continue;
- break;
- }
- }
-
- // copy repeated sequence
- if unlikely((op-ref)<(int)STEPSIZE)
- {
- const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref];
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- op += 4, ref += 4; ref -= dec32table[op-ref];
- A32(op) = A32(ref);
- op += STEPSIZE-4; ref -= dec64;
- } else { LZ4_COPYSTEP(op,ref); }
- cpy = op + length - (STEPSIZE-4);
-
- if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4))
- {
- if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
- LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));
- while(op<cpy) *op++=*ref++;
- op=cpy;
- continue;
- }
- LZ4_WILDCOPY(op, ref, cpy);
- op=cpy; // correction
- }
-
- // end of decoding
- if (endOnInput)
- return (int) (((char*)op)-dest); // Nb of output bytes decoded
- else
- return (int) (((char*)ip)-source); // Nb of input bytes read
-
- // Overflow error detected
-_output_error:
- return (int) (-(((char*)ip)-source))-1;
-}
-
-
-int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, full, 0);
-}
-
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, withPrefix, full, 0);
-}
-
-int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize)
-{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize);
-}
-
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)
-{
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
-}
-
-int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
-{
-#ifdef _MSC_VER // This version is faster with Visual
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0);
-#else
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
-#endif
-}
-
+/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +//************************************** +// Tuning parameters +//************************************** +// MEMORY_USAGE : +// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +// Increasing memory usage improves compression ratio +// Reduced memory usage can improve speed, due to cache effect +// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache +#define MEMORY_USAGE 14 + +// HEAPMODE : +// Select how default compression functions will allocate memory for their hash table, +// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). +#define HEAPMODE 0 + + +//************************************** +// CPU Feature Detection +//************************************** +// 32 or 64 bits ? +#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ + || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ + || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode +# define LZ4_ARCH64 1 +#else +# define LZ4_ARCH64 0 +#endif + +// Little Endian or Big Endian ? +// Overwrite the #define below if you know your architecture endianess +#if defined (__GLIBC__) +# include <endian.h> +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define LZ4_BIG_ENDIAN 1 +# endif +#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) +# define LZ4_BIG_ENDIAN 1 +#elif defined(__sparc) || defined(__sparc__) \ + || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) +# define LZ4_BIG_ENDIAN 1 +#else +// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +#endif + +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +#if defined(__ARM_FEATURE_UNALIGNED) +# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#endif + +// Define this parameter if your target system or compiler does not support hardware bit count +#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +# define LZ4_FORCE_SW_BITCOUNT +#endif + +// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : +// This option may provide a small boost to performance for some big endian cpu, although probably modest. +// You may set this option to 1 if data will remain within closed environment. +// This option is useless on Little_Endian CPU (such as x86) +//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 + + +//************************************** +// Compiler Options +//************************************** +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99 +/* "restrict" is a known keyword */ +#else +# define restrict // Disable restrict +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +# include <intrin.h> // For Visual 2005 +# if LZ4_ARCH64 // 64-bits +# pragma intrinsic(_BitScanForward64) // For Visual 2005 +# pragma intrinsic(_BitScanReverse64) // For Visual 2005 +# else // 32-bits +# pragma intrinsic(_BitScanForward) // For Visual 2005 +# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# endif +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +#ifdef _MSC_VER +# define lz4_bswap16(x) _byteswap_ushort(x) +#else +# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +//************************************** +// Memory routines +//************************************** +#include <stdlib.h> // malloc, calloc, free +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include <string.h> // memset, memcpy +#define MEM_INIT memset + + +//************************************** +// Includes +//************************************** +#include "lz4.h" + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct { U16 v; } _PACKED U16_S; +typedef struct { U32 v; } _PACKED U32_S; +typedef struct { U64 v; } _PACKED U64_S; +typedef struct {size_t v;} _PACKED size_t_S; + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(0) +# else +# pragma pack(pop) +# endif +#endif + +#define A16(x) (((U16_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) +#define AARCH(x) (((size_t_S *)(x))->v) + + +//************************************** +// Constants +//************************************** +#define LZ4_HASHLOG (MEMORY_USAGE-2) +#define HASHTABLESIZE (1 << MEMORY_USAGE) +#define HASHNBCELLS4 (1 << LZ4_HASHLOG) + +#define MINMATCH 4 + +#define COPYLENGTH 8 +#define LASTLITERALS 5 +#define MFLIMIT (COPYLENGTH+MINMATCH) +const int LZ4_minLength = (MFLIMIT+1); + +#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1)) +#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data + +#define MAXD_LOG 16 +#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) + +#define ML_BITS 4 +#define ML_MASK ((1U<<ML_BITS)-1) +#define RUN_BITS (8-ML_BITS) +#define RUN_MASK ((1U<<RUN_BITS)-1) + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + + +//************************************** +// Structures and local types +//************************************** + +typedef struct { + U32 hashTable[HASHNBCELLS4]; + const BYTE* bufferStart; + const BYTE* base; + const BYTE* nextBlock; +} LZ4_Data_Structure; + +typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +//************************************** +// Architecture-specific macros +//************************************** +#define STEPSIZE sizeof(size_t) +#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; } +#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); } +#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); } + +#if LZ4_ARCH64 // 64-bit +# define HTYPE U32 +# define INITBASE(base) const BYTE* const base = ip +#else // 32-bit +# define HTYPE const BYTE* +# define INITBASE(base) const int base = 0 +#endif + +#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE)) +# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } +# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; } +#else // Little Endian +# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); } +# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } +#endif + + +//************************************** +// Macros +//************************************** +#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e; + + +//**************************** +// Private functions +//**************************** +#if LZ4_ARCH64 + +FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) +{ +# if defined(LZ4_BIG_ENDIAN) +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); +# else + int r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif +# else +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll(val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif +# endif +} + +#else + +FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) +{ +# if defined(LZ4_BIG_ENDIAN) +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); +# else + int r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif +# else +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz(val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif +# endif +} + +#endif + + +//**************************** +// Compression functions +//**************************** +FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType) +{ + if (tableType == byU16) + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); } + +FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; } + } +} + +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return +} + +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + + +FORCE_INLINE int LZ4_compress_generic( + void* ctx, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + + limitedOutput_directive limitedOutput, + tableType_t tableType, + prefix64k_directive prefix) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source; + const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source); + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + int length; + const int skipStrength = SKIPSTRENGTH; + U32 forwardH; + + // Init conditions + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative) + if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block + if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit + if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit) + if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals) + + // First Byte + LZ4_putPosition(ip, ctx, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + // Main Loop + for ( ; ; ) + { + int findMatchAttempts = (1U << skipStrength) + 3; + const BYTE* forwardIp = ip; + const BYTE* ref; + BYTE* token; + + // Find a match + do { + U32 h = forwardH; + int step = findMatchAttempts++ >> skipStrength; + ip = forwardIp; + forwardIp = ip + step; + + if unlikely(forwardIp > mflimit) { goto _last_literals; } + + forwardH = LZ4_hashPosition(forwardIp, tableType); + ref = LZ4_getPositionOnHash(h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip))); + + // Catch up + while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; } + + // Encode Literal length + length = (int)(ip - anchor); + token = op++; + if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit + if (length>=(int)RUN_MASK) + { + int len = length-RUN_MASK; + *token=(RUN_MASK<<ML_BITS); + for(; len >= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(length<<ML_BITS); + + // Copy Literals + { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; } + +_next_match: + // Encode Offset + LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref)); + + // Start Counting + ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified + anchor = ip; + while likely(ip<matchlimit-(STEPSIZE-1)) + { + size_t diff = AARCH(ref) ^ AARCH(ip); + if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; } + ip += LZ4_NbCommonBytes(diff); + goto _endCount; + } + if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; } + if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; } + if ((ip<matchlimit) && (*ref == *ip)) ip++; +_endCount: + + // Encode MatchLength + length = (int)(ip - anchor); + if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit + if (length>=(int)ML_MASK) + { + *token += ML_MASK; + length -= ML_MASK; + for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length-=255; *op++ = 255; } + *op++ = (BYTE)length; + } + else *token += (BYTE)(length); + + // Test end of chunk + if (ip > mflimit) { anchor = ip; break; } + + // Fill table + LZ4_putPosition(ip-2, ctx, tableType, base); + + // Test next position + ref = LZ4_getPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx, tableType, base); + if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } + + // Prepare next loop + anchor = ip++; + forwardH = LZ4_hashPosition(ip, tableType); + } + +_last_literals: + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); + if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<<ML_BITS); + memcpy(op, anchor, iend - anchor); + op += iend-anchor; + } + + // End + return (int) (((char*)op)-dest); +} + + +int LZ4_compress(const char* source, char* dest, int inputSize) +{ +#if (HEAPMODE) + void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries +#else + U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries +#endif + int result; + + if (inputSize < (int)LZ4_64KLIMIT) + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noPrefix); + else + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ +#if (HEAPMODE) + void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries +#else + U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries +#endif + int result; + + if (inputSize < (int)LZ4_64KLIMIT) + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix); + else + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + +//***************************** +// Using an external allocation +//***************************** + +int LZ4_sizeofState() { return 1 << MEMORY_USAGE; } + + +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) +{ + if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary + MEM_INIT(state, 0, LZ4_sizeofState()); + + if (inputSize < (int)LZ4_64KLIMIT) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noPrefix); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix); +} + + +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary + MEM_INIT(state, 0, LZ4_sizeofState()); + + if (inputSize < (int)LZ4_64KLIMIT) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix); +} + + +//**************************** +// Stream functions +//**************************** + +int LZ4_sizeofStreamState() +{ + return sizeof(LZ4_Data_Structure); +} + +FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base) +{ + MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable)); + lz4ds->bufferStart = base; + lz4ds->base = base; + lz4ds->nextBlock = base; +} + +int LZ4_resetStreamState(void* state, const char* inputBuffer) +{ + if ((((size_t)state) & 3) != 0) return 1; // Error : pointer is not aligned on 4-bytes boundary + LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (const char* inputBuffer) +{ + void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure)); + LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer); + return lz4ds; +} + + +int LZ4_free (void* LZ4_Data) +{ + FREEMEM(LZ4_Data); + return (0); +} + + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data; + size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB); + + if ( (lz4ds->base - delta > lz4ds->base) // underflow control + || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit + { + size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base; + int nH; + + for (nH=0; nH < HASHNBCELLS4; nH++) + { + if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0; + else lz4ds->hashTable[nH] -= (U32)deltaLimit; + } + memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB); + lz4ds->base = lz4ds->bufferStart; + lz4ds->nextBlock = lz4ds->base + 64 KB; + } + else + { + memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB); + lz4ds->nextBlock -= delta; + lz4ds->base -= delta; + } + + return (char*)(lz4ds->nextBlock); +} + + +int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix); +} + + +int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix); +} + + +//**************************** +// Decompression functions +//**************************** + +// This generic decompression function cover all use cases. +// It shall be instanciated several times, using different sets of directives +// Note that it is essential this generic function is really inlined, +// in order to remove useless branches during compilation optimisation. +FORCE_INLINE int LZ4_decompress_generic( + const char* source, + char* dest, + int inputSize, // + int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer. + + int endOnInput, // endOnOutputSize, endOnInputSize + int prefix64k, // noPrefix, withPrefix + int partialDecoding, // full, partial + int targetOutputSize // only used if partialDecoding==partial + ) +{ + // Local Variables + const BYTE* restrict ip = (const BYTE*) source; + const BYTE* ref; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + + const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64 + static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; + + + // Special cases + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything + if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer + if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1); + + + // Main Loop + while (1) + { + unsigned token; + size_t length; + + // get runlength + token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) + { + unsigned s=255; + while (((endOnInput)?ip<iend:1) && (s==255)) + { + s = *ip++; + length += s; + } + } + + // copy literals + cpy = op+length; + if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-COPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer + if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed + } + memcpy(op, ip, length); + ip += length; + op += length; + break; // Necessarily EOF, due to parsing restrictions + } + LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; + + // get offset + LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; + if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer + + // get matchlength + if ((length=(token&ML_MASK)) == ML_MASK) + { + while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token + { + unsigned s = *ip++; + length += s; + if (s==255) continue; + break; + } + } + + // copy repeated sequence + if unlikely((op-ref)<(int)STEPSIZE) + { + const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref]; + op[0] = ref[0]; + op[1] = ref[1]; + op[2] = ref[2]; + op[3] = ref[3]; + op += 4, ref += 4; ref -= dec32table[op-ref]; + A32(op) = A32(ref); + op += STEPSIZE-4; ref -= dec64; + } else { LZ4_COPYSTEP(op,ref); } + cpy = op + length - (STEPSIZE-4); + + if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals + LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH)); + while(op<cpy) *op++=*ref++; + op=cpy; + continue; + } + LZ4_WILDCOPY(op, ref, cpy); + op=cpy; // correction + } + + // end of decoding + if (endOnInput) + return (int) (((char*)op)-dest); // Nb of output bytes decoded + else + return (int) (((char*)ip)-source); // Nb of input bytes read + + // Overflow error detected +_output_error: + return (int) (-(((char*)ip)-source))-1; +} + + +int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, full, 0); +} + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, withPrefix, full, 0); +} + +int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0); +} + +int LZ4_decompress_fast(const char* source, char* dest, int outputSize) +{ +#ifdef _MSC_VER // This version is faster with Visual + return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0); +#else + return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0); +#endif +} + @@ -1,205 +1,249 @@ -/*
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio
-# define inline __inline // Visual C is not C99, but supports some kind of inline
-#endif
-
-
-//****************************
-// Simple Functions
-//****************************
-
-int LZ4_compress (const char* source, char* dest, int inputSize);
-int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-LZ4_compress() :
- Compresses 'inputSize' bytes from 'source' into 'dest'.
- Destination buffer must be already allocated,
- and must be sized to handle worst cases situations (input data not compressible)
- Worst case size evaluation is provided by function LZ4_compressBound()
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
- return : the number of bytes written in buffer dest
- or 0 if the compression fails
-
-LZ4_decompress_safe() :
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
-*/
-
-
-//****************************
-// Advanced Functions
-//****************************
-#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes
-#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
-static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-
-/*
-LZ4_compressBound() :
- Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
- primarily useful for memory allocation of output buffer.
- inline function is recommended for the general case,
- macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
-
- isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
- return : maximum output size in a "worst case" scenario
- or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
-*/
-
-
-int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-LZ4_compress_limitedOutput() :
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
- If it cannot achieve it, compression will stop, and result of the function will be zero.
- This function never writes outside of provided output buffer.
-
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes written in buffer 'dest'
- or 0 if the compression fails
-*/
-
-
-int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
-
-/*
-LZ4_decompress_fast() :
- outputSize : is the original (uncompressed) size
- return : the number of bytes read from the source buffer (in other words, the compressed size)
- If the source stream is malformed, the function will stop decoding and return a negative result.
- note : This function is a bit faster than LZ4_decompress_safe()
- This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
- Use this function preferably into a trusted environment (data to decode comes from a trusted source).
- Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
-*/
-
-int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
-
-/*
-LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'inputSize' at position 'source'
- into output buffer 'dest' of size 'maxOutputSize'.
- The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
- reducing decompression time.
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
- Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
- Always control how many bytes were decoded.
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
-*/
-
-
-//****************************
-// Stream Functions
-//****************************
-
-void* LZ4_create (const char* inputBuffer);
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBuffer (void* LZ4_Data);
-int LZ4_free (void* LZ4_Data);
-
-/*
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
-In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
-
-void* LZ4_create (const char* inputBuffer);
-The result of the function is the (void*) pointer on the LZ4 Data Structure.
-This pointer will be needed in all other functions.
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
-To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
-Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
-but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
-If next block does not begin immediately after the previous one, the compression will fail (return 0).
-
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
-char* LZ4_slideInputBuffer(void* LZ4_Data);
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
-==> The memory position where the next input data block must start is provided as the result of the function.
-
-Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
-
-When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
-*/
-
-
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
-
-/*
-*_withPrefix64k() :
- These decoding functions work the same as their "normal name" versions,
- but can use up to 64KB of data in front of 'char* dest'.
- These functions are necessary to decode inter-dependant blocks.
-*/
-
-
-//****************************
-// Obsolete Functions
-//****************************
-
-static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
-
-/*
-These functions are deprecated and should no longer be used.
-They are provided here for compatibility with existing user programs.
-*/
-
-
-
-#if defined (__cplusplus)
-}
-#endif
+/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +//************************************** +// Compiler Options +//************************************** +#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio +# define inline __inline // Visual C is not C99, but supports some kind of inline +#endif + + +//**************************** +// Simple Functions +//**************************** + +int LZ4_compress (const char* source, char* dest, int inputSize); +int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize); + +/* +LZ4_compress() : + Compresses 'inputSize' bytes from 'source' into 'dest'. + Destination buffer must be already allocated, + and must be sized to handle worst cases situations (input data not compressible) + Worst case size evaluation is provided by function LZ4_compressBound() + inputSize : Max supported value is LZ4_MAX_INPUT_VALUE + return : the number of bytes written in buffer dest + or 0 if the compression fails + +LZ4_decompress_safe() : + maxOutputSize : is the size of the destination buffer (which must be already allocated) + return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets +*/ + + +//**************************** +// Advanced Functions +//**************************** +#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes +#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } + +/* +LZ4_compressBound() : + Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) + primarily useful for memory allocation of output buffer. + inline function is recommended for the general case, + macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation). + + isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ + + +int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); + +/* +LZ4_compress_limitedOutput() : + Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. + If it cannot achieve it, compression will stop, and result of the function will be zero. + This function never writes outside of provided output buffer. + + inputSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxOutputSize : is the size of the destination buffer (which must be already allocated) + return : the number of bytes written in buffer 'dest' + or 0 if the compression fails +*/ + + +int LZ4_decompress_fast (const char* source, char* dest, int outputSize); + +/* +LZ4_decompress_fast() : + outputSize : is the original (uncompressed) size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is malformed, the function will stop decoding and return a negative result. + note : This function is a bit faster than LZ4_decompress_safe() + This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet. + Use this function preferably into a trusted environment (data to decode comes from a trusted source). + Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes. +*/ + +int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize); + +/* +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'inputSize' at position 'source' + into output buffer 'dest' of size 'maxOutputSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ + + +//***************************** +// Using an external allocation +//***************************** +int LZ4_sizeofState(); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* +These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. +To know how much memory must be allocated for the compression tables, use : +int LZ4_sizeofState(); + +Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0). + +The allocated memory can be provided to the compressions functions using 'void* state' parameter. +LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions. +They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). +*/ + + +//**************************** +// Streaming Functions +//**************************** + +void* LZ4_create (const char* inputBuffer); +int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize); +char* LZ4_slideInputBuffer (void* LZ4_Data); +int LZ4_free (void* LZ4_Data); + +/* +These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. +In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function : + +void* LZ4_create (const char* inputBuffer); +The result of the function is the (void*) pointer on the LZ4 Data Structure. +This pointer will be needed in all other functions. +If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. +The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. +To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(). +Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(), +but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one. +If next block does not begin immediately after the previous one, the compression will fail (return 0). + +When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : +char* LZ4_slideInputBuffer(void* LZ4_Data); +must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. +Note that, for this function to work properly, minimum size of an input buffer must be 192KB. +==> The memory position where the next input data block must start is provided as the result of the function. + +Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual. + +When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure. +*/ + +int LZ4_sizeofStreamState(); +int LZ4_resetStreamState(void* state, const char* inputBuffer); + +/* +These functions achieve the same result as : +void* LZ4_create (const char* inputBuffer); + +They are provided here to allow the user program to allocate memory using its own routines. + +To know how much space must be allocated, use LZ4_sizeofStreamState(); +Note also that space must be 4-bytes aligned. + +Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer); +void* state is a pointer to the space allocated. +It must be aligned on 4-bytes boundaries, and be large enough. +The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). +return value of LZ4_resetStreamState() must be 0 is OK. +Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries). +*/ + + +int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize); + +/* +*_withPrefix64k() : + These decoding functions work the same as their "normal name" versions, + but can use up to 64KB of data in front of 'char* dest'. + These functions are necessary to decode inter-dependant blocks. +*/ + + +//**************************** +// Obsolete Functions +//**************************** + +static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + +/* +These functions are deprecated and should no longer be used. +They are provided here for compatibility with existing user programs. +*/ + + + +#if defined (__cplusplus) +} +#endif @@ -1,1251 +1,1257 @@ -/*
- LZ4cli.c - LZ4 Command Line Interface
- Copyright (C) Yann Collet 2011-2013
- GPL v2 License
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
- You can contact the author at :
- - LZ4 source repository : http://code.google.com/p/lz4/
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-/*
- Note : this is stand-alone program.
- It is not part of LZ4 compression library, it is a user program of the LZ4 library.
- The license of LZ4 library is BSD.
- The license of xxHash library is BSD.
- The license of this compression CLI program is GPLv2.
-*/
-
-//**************************************
-// Tuning parameters
-//**************************************
-// DISABLE_LZ4C_LEGACY_OPTIONS :
-// Control the availability of -c0, -c1 and -hc legacy arguments
-// Default : Legacy options are enabled
-// #define DISABLE_LZ4C_LEGACY_OPTIONS
-
-
-//**************************************
-// Compiler Options
-//**************************************
-// Disable some Visual warning messages
-#ifdef _MSC_VER // Visual Studio
-# define _CRT_SECURE_NO_WARNINGS
-# define _CRT_SECURE_NO_DEPRECATE // VS2005
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-#endif
-
-#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix
-#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix
-
-
-//****************************
-// Includes
-//****************************
-#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout
-#include <stdlib.h> // malloc
-#include <string.h> // strcmp, strlen
-#include <time.h> // clock
-#include "lz4.h"
-#include "lz4hc.h"
-#include "xxhash.h"
-#include "bench.h"
-
-
-//****************************
-// OS-specific Includes
-//****************************
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-# include <fcntl.h> // _O_BINARY
-# include <io.h> // _setmode, _isatty
-# ifdef __MINGW32__
- int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h>
-# endif
-# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
-# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#else
-# include <unistd.h> // isatty
-# define SET_BINARY_MODE(file)
-# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#endif
-
-
-//**************************************
-// Compiler-specific functions
-//**************************************
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-#if defined(_MSC_VER) // Visual Studio
-# define swap32 _byteswap_ulong
-#elif GCC_VERSION >= 403
-# define swap32 __builtin_bswap32
-#else
- static inline unsigned int swap32(unsigned int x)
- {
- return ((x << 24) & 0xff000000 ) |
- ((x << 8) & 0x00ff0000 ) |
- ((x >> 8) & 0x0000ff00 ) |
- ((x >> 24) & 0x000000ff );
- }
-#endif
-
-
-//****************************
-// Constants
-//****************************
-#define COMPRESSOR_NAME "LZ4 Compression CLI"
-#ifndef LZ4_VERSION
-# define LZ4_VERSION "v1.0.9"
-#endif
-#define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
-#define LZ4_EXTENSION ".lz4"
-
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-#define _1BIT 0x01
-#define _2BITS 0x03
-#define _3BITS 0x07
-#define _4BITS 0x0F
-#define _8BITS 0xFF
-
-#define MAGICNUMBER_SIZE 4
-#define LZ4S_MAGICNUMBER 0x184D2204
-#define LZ4S_SKIPPABLE0 0x184D2A50
-#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
-#define LEGACY_MAGICNUMBER 0x184C2102
-
-#define CACHELINE 64
-#define LEGACY_BLOCKSIZE (8 MB)
-#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
-#define LZ4S_BLOCKSIZEID_DEFAULT 7
-#define LZ4S_CHECKSUM_SEED 0
-#define LZ4S_EOS 0
-#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
-
-
-//**************************************
-// Architecture Macros
-//**************************************
-static const int one = 1;
-#define CPU_LITTLE_ENDIAN (*(char*)(&one))
-#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)
-#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i))
-
-
-//**************************************
-// Macros
-//**************************************
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-
-
-//**************************************
-// Special input/output
-//**************************************
-#define NULL_OUTPUT "null"
-char stdinmark[] = "stdin";
-char stdoutmark[] = "stdout";
-#ifdef _WIN32
-char nulmark[] = "nul";
-#else
-char nulmark[] = "/dev/null";
-#endif
-
-
-//**************************************
-// Local Parameters
-//**************************************
-static char* programName;
-static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information
-static int overwrite = 0;
-static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
-static int blockChecksum = 0;
-static int streamChecksum = 1;
-static int blockIndependence = 1;
-
-
-//**************************************
-// Exceptions
-//**************************************
-#define DEBUG 0
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...) \
-{ \
- DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
- DISPLAYLEVEL(1, "Error %i : ", error); \
- DISPLAYLEVEL(1, __VA_ARGS__); \
- DISPLAYLEVEL(1, "\n"); \
- exit(error); \
-}
-
-
-//**************************************
-// Version modifiers
-//**************************************
-#define EXTENDED_ARGUMENTS
-#define EXTENDED_HELP
-#define EXTENDED_FORMAT
-#define DEFAULT_COMPRESSOR compress_file
-#define DEFAULT_DECOMPRESSOR decodeLZ4S
-
-
-//****************************
-// Functions
-//****************************
-int usage()
-{
- DISPLAY( "Usage :\n");
- DISPLAY( " %s [arg] [input] [output]\n", programName);
- DISPLAY( "\n");
- DISPLAY( "input : a filename\n");
- DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
- DISPLAY( "Arguments :\n");
- DISPLAY( " -1 : Fast compression (default) \n");
- DISPLAY( " -9 : High compression \n");
- DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);
- DISPLAY( " -z : force compression\n");
- DISPLAY( " -f : overwrite output without prompting \n");
- DISPLAY( " -h/-H : display help/long help and exit\n");
- return 0;
-}
-
-int usage_advanced()
-{
- DISPLAY(WELCOME_MESSAGE);
- usage();
- DISPLAY( "\n");
- DISPLAY( "Advanced arguments :\n");
- DISPLAY( " -V : display Version number and exit\n");
- DISPLAY( " -v : verbose mode\n");
- DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
- DISPLAY( " -c : force write to standard output, even if it is the console\n");
- DISPLAY( " -t : test compressed file integrity\n");
- DISPLAY( " -B# : Block size [4-7](default : 7)\n");
- DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
- DISPLAY( " -BX : enable block checksum (default:disabled)\n");
- DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");
- DISPLAY( "Benchmark arguments :\n");
- DISPLAY( " -b : benchmark file(s)\n");
- DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
- DISPLAY( "Legacy arguments :\n");
- DISPLAY( " -c0 : fast compression\n");
- DISPLAY( " -c1 : high compression\n");
- DISPLAY( " -hc : high compression\n");
- DISPLAY( " -y : overwrite output without prompting \n");
- DISPLAY( " -s : suppress warnings \n");
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS
- EXTENDED_HELP;
- return 0;
-}
-
-int usage_longhelp()
-{
- DISPLAY( "\n");
- DISPLAY( "Which values can get [output] ? \n");
- DISPLAY( "[output] : a filename\n");
- DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark);
- DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT);
- DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n");
- DISPLAY( " - if stdout is not the console, then [output] = stdout \n");
- DISPLAY( " - if stdout is console : \n");
- DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION);
- DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION);
- DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION);
- DISPLAY( "\n");
- DISPLAY( "Compression levels : \n");
- DISPLAY( "There are technically 2 accessible compression levels.\n");
- DISPLAY( "-0 ... -2 => Fast compression\n");
- DISPLAY( "-3 ... -9 => High compression\n");
- DISPLAY( "\n");
- DISPLAY( "stdin, stdout and the console : \n");
- DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");
- DISPLAY( "%s will refuse to read from console, or write to console \n", programName);
- DISPLAY( "except if '-c' command is specified, to force output to console \n");
- DISPLAY( "\n");
- DISPLAY( "Simple example :\n");
- DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");
- DISPLAY( " %s filename\n", programName);
- DISPLAY( "\n");
- DISPLAY( "Arguments can be appended together, or provided independently. For example :\n");
- DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");
- DISPLAY( " %s -f9 filename \n", programName);
- DISPLAY( " is equivalent to :\n");
- DISPLAY( " %s -f -9 filename \n", programName);
- DISPLAY( "\n");
- DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName);
- DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
- DISPLAY( " generator | %s | consumer \n", programName);
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
- DISPLAY( "\n");
- DISPLAY( "Warning :\n");
- DISPLAY( "Legacy arguments take precedence. Therefore : \n");
- DISPLAY( " %s -hc filename\n", programName);
- DISPLAY( "means 'compress filename in high compression mode'\n");
- DISPLAY( "It is not equivalent to :\n");
- DISPLAY( " %s -h -c filename\n", programName);
- DISPLAY( "which would display help text and exit\n");
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS
- return 0;
-}
-
-int badusage()
-{
- DISPLAYLEVEL(1, "Incorrect parameters\n");
- if (displayLevel >= 1) usage();
- exit(1);
-}
-
-
-static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
-static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }
-static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
-
-
-int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
-{
-
- if (!strcmp (input_filename, stdinmark))
- {
- DISPLAYLEVEL(4,"Using stdin for input\n");
- *pfinput = stdin;
- SET_BINARY_MODE(stdin);
- }
- else
- {
- *pfinput = fopen(input_filename, "rb");
- }
-
- if (!strcmp (output_filename, stdoutmark))
- {
- DISPLAYLEVEL(4,"Using stdout for output\n");
- *pfoutput = stdout;
- SET_BINARY_MODE(stdout);
- }
- else
- {
- // Check if destination file already exists
- *pfoutput=0;
- if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );
- if (*pfoutput!=0)
- {
- fclose(*pfoutput);
- if (!overwrite)
- {
- char ch;
- DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
- DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
- if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible
- ch = (char)getchar();
- if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
- }
- }
- *pfoutput = fopen( output_filename, "wb" );
- }
-
- if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
- if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
-
- return 0;
-}
-
-
-
-int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel)
-{
- int (*compressionFunction)(const char*, char*, int);
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
- char* in_buff;
- char* out_buff;
- FILE* finput;
- FILE* foutput;
- int displayLevel = (compressionlevel>0);
- clock_t start, end;
- size_t sizeCheck;
-
-
- // Init
- if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
- start = clock();
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
-
- // Allocate Memory
- in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
- if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
-
- // Write Archive Header
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);
- sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");
-
- // Main Loop
- while (1)
- {
- unsigned int outSize;
- // Read Block
- int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
- if( inSize<=0 ) break;
- filesize += inSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
-
- // Compress Block
- outSize = compressionFunction(in_buff, out_buff+4, inSize);
- compressedfilesize += outSize+4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
- if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");
- }
-
- // Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close & Free
- free(in_buff);
- free(out_buff);
- fclose(finput);
- fclose(foutput);
-
- return 0;
-}
-
-
-int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
-{
- void* (*initFunction) (const char*);
- int (*compressionFunction)(void*, const char*, char*, int, int);
- char* (*translateFunction) (void*);
- int (*freeFunction) (void*);
- void* ctx;
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = 0;
- unsigned int checkbits;
- char* in_buff, *in_start, *in_end;
- char* out_buff;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
- unsigned int blockSize, inputBufferSize;
- size_t sizeCheck, header_size;
- void* streamChecksumState=NULL;
-
-
- // Init
- start = clock();
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
- if (compressionlevel>=3)
- {
- initFunction = LZ4_createHC;
- compressionFunction = LZ4_compressHC_limitedOutput_continue;
- translateFunction = LZ4_slideInputBufferHC;
- freeFunction = LZ4_freeHC;
- }
- else
- {
- initFunction = LZ4_create;
- compressionFunction = LZ4_compress_limitedOutput_continue;
- translateFunction = LZ4_slideInputBuffer;
- freeFunction = LZ4_free;
- }
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
-
- // Allocate Memory
- inputBufferSize = blockSize + 64 KB;
- if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
- in_buff = (char*)malloc(inputBufferSize);
- out_buff = (char*)malloc(blockSize+CACHELINE);
- if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
- in_start = in_buff; in_end = in_buff + inputBufferSize;
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
- ctx = initFunction(in_buff);
-
- // Write Archive Header
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
- *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')
- *(out_buff+4) |= (blockIndependence & _1BIT) << 5;
- *(out_buff+4) |= (blockChecksum & _1BIT) << 4;
- *(out_buff+4) |= (streamChecksum & _1BIT) << 2;
- *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);
- checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
- *(out_buff+6) = (unsigned char) checkbits;
- header_size = 7;
- sizeCheck = fwrite(out_buff, 1, header_size, foutput);
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
- compressedfilesize += header_size;
-
- // Main Loop
- while (1)
- {
- unsigned int outSize;
- unsigned int inSize;
- // Read Block
- if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
- inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
- if( inSize==0 ) break; // No more input : end of compression
- filesize += inSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
- if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
-
- // Compress Block
- outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
- if (blockChecksum) compressedfilesize+=4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- if (outSize > 0)
- {
- int sizeToWrite;
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
- }
- sizeToWrite = 4 + outSize + (4*blockChecksum);
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
-
- }
- else // Copy Original
- {
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
- sizeCheck = fwrite(in_start, 1, inSize, foutput);
- if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
- }
- }
- in_start += inSize;
- }
-
- // End of Stream mark
- * (unsigned int*) out_buff = LZ4S_EOS;
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
- compressedfilesize += 4;
- if (streamChecksum)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
- compressedfilesize += 4;
- }
-
- // Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close & Free
- freeFunction(ctx);
- free(in_buff);
- free(out_buff);
- fclose(finput);
- fclose(foutput);
-
- return 0;
-}
-
-
-int compress_file(char* input_filename, char* output_filename, int compressionlevel)
-{
- int (*compressionFunction)(const char*, char*, int, int);
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = 0;
- unsigned int checkbits;
- char* in_buff;
- char* out_buff;
- char* headerBuffer;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
- int blockSize;
- size_t sizeCheck, header_size, readSize;
- void* streamChecksumState=NULL;
-
- // Branch out
- if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);
-
- // Init
- start = clock();
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
- if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput;
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
-
- // Allocate Memory
- in_buff = (char*)malloc(blockSize);
- out_buff = (char*)malloc(blockSize+CACHELINE);
- headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE);
- if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory");
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
-
- // Write Archive Header
- *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
- *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01')
- *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5;
- *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4;
- *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2;
- *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4);
- checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED);
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
- *(headerBuffer+6) = (unsigned char) checkbits;
- header_size = 7;
-
- // Write header
- sizeCheck = fwrite(headerBuffer, 1, header_size, foutput);
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
- compressedfilesize += header_size;
-
- // read first block
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
-
- // Main Loop
- while (readSize>0)
- {
- unsigned int outSize;
-
- filesize += readSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
- if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize);
-
- // Compress Block
- outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1);
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
- if (blockChecksum) compressedfilesize+=4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- if (outSize > 0)
- {
- int sizeToWrite;
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
- }
- sizeToWrite = 4 + outSize + (4*blockChecksum);
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
- }
- else // Copy Original Uncompressed
- {
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
- sizeCheck = fwrite(in_buff, 1, readSize, foutput);
- if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block");
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
- }
- }
-
- // Read next block
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
- }
-
- // End of Stream mark
- * (unsigned int*) out_buff = LZ4S_EOS;
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
- compressedfilesize += 4;
- if (streamChecksum)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
- compressedfilesize += 4;
- }
-
- // Close & Free
- free(in_buff);
- free(out_buff);
- free(headerBuffer);
- fclose(finput);
- fclose(foutput);
-
- // Final Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- return 0;
-}
-
-
-unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
-{
- unsigned long long filesize = 0;
- char* in_buff;
- char* out_buff;
- unsigned int blockSize;
-
-
- // Allocate Memory
- in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
- out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
-
- // Main Loop
- while (1)
- {
- int decodeSize;
- size_t sizeCheck;
-
- // Block Size
- sizeCheck = fread(&blockSize, 1, 4, finput);
- if (sizeCheck==0) break; // Nothing to read : file read is completed
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian
- if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
- { // Cannot read next block : maybe new stream ?
- fseek(finput, -4, SEEK_CUR);
- break;
- }
-
- // Read Block
- sizeCheck = fread(in_buff, 1, blockSize, finput);
-
- // Decode Block
- decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
- if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
- filesize += decodeSize;
-
- // Write Block
- sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
- if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
- }
-
- // Free
- free(in_buff);
- free(out_buff);
-
- return filesize;
-}
-
-
-unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
-{
- unsigned long long filesize = 0;
- char* in_buff;
- char* out_buff, *out_start, *out_end;
- unsigned char descriptor[LZ4S_MAXHEADERSIZE];
- size_t nbReadBytes;
- int decodedBytes=0;
- unsigned int maxBlockSize;
- size_t sizeCheck;
- int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
- void* streamChecksumState=NULL;
- int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
- unsigned int prefix64k = 0;
-
- // Decode stream descriptor
- nbReadBytes = fread(descriptor, 1, 3, finput);
- if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");
- {
- int version = (descriptor[0] >> 6) & _2BITS;
- int streamSize = (descriptor[0] >> 3) & _1BIT;
- int reserved1 = (descriptor[0] >> 1) & _1BIT;
- int dictionary = (descriptor[0] >> 0) & _1BIT;
-
- int reserved2 = (descriptor[1] >> 7) & _1BIT;
- int blockSizeId = (descriptor[1] >> 4) & _3BITS;
- int reserved3 = (descriptor[1] >> 0) & _4BITS;
- int checkBits = (descriptor[2] >> 0) & _8BITS;
- int checkBits_xxh32;
-
- blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;
- blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;
- streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;
-
- if (version != 1) EXM_THROW(62, "Wrong version number");
- if (streamSize == 1) EXM_THROW(64, "Does not support stream size");
- if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");
- if (dictionary == 1) EXM_THROW(66, "Does not support dictionary");
- if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");
- if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size");
- if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");
- maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);
- // Checkbits verification
- descriptor[1] &= 0xF0;
- checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);
- checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);
- if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
- }
-
- if (!blockIndependenceFlag)
- {
- decompressionFunction = LZ4_decompress_safe_withPrefix64k;
- prefix64k = 64 KB;
- }
-
- // Allocate Memory
- {
- unsigned int outbuffSize = prefix64k+maxBlockSize;
- in_buff = (char*)malloc(maxBlockSize);
- if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
- out_buff = (char*)malloc(outbuffSize);
- out_end = out_buff + outbuffSize;
- out_start = out_buff + prefix64k;
- if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
- }
- if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
-
- // Main Loop
- while (1)
- {
- unsigned int blockSize, uncompressedFlag;
-
- // Block Size
- nbReadBytes = fread(&blockSize, 1, 4, finput);
- if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");
- if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian
- uncompressedFlag = blockSize >> 31;
- blockSize &= 0x7FFFFFFF;
- if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");
-
- // Read Block
- nbReadBytes = fread(in_buff, 1, blockSize, finput);
- if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );
-
- // Check Block
- if (blockChecksumFlag)
- {
- unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);
- unsigned int readChecksum;
- sizeCheck = fread(&readChecksum, 1, 4, finput);
- if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");
- }
-
- if (uncompressedFlag)
- {
- // Write uncompressed Block
- sizeCheck = fwrite(in_buff, 1, blockSize, foutput);
- if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");
- filesize += blockSize;
- if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
- if (!blockIndependenceFlag)
- {
- if (blockSize >= prefix64k)
- {
- memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
- out_start = out_buff + prefix64k;
- continue;
- }
- else
- {
- memcpy(out_start, in_buff, blockSize);
- decodedBytes = blockSize;
- }
- }
- }
- else
- {
- // Decode Block
- decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
- if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
- filesize += decodedBytes;
- if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
-
- // Write Block
- sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
- if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
- }
-
- if (!blockIndependenceFlag)
- {
- out_start += decodedBytes;
- if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
- {
- memcpy(out_buff, out_start - prefix64k, prefix64k);
- out_start = out_buff + prefix64k;
- }
- }
- }
-
- // Stream Checksum
- if (streamChecksumFlag)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- unsigned int readChecksum;
- sizeCheck = fread(&readChecksum, 1, 4, finput);
- if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
- }
-
- // Free
- free(in_buff);
- free(out_buff);
-
- return filesize;
-}
-
-
-unsigned long long selectDecoder( FILE* finput, FILE* foutput)
-{
- unsigned int magicNumber, size;
- int errorNb;
- size_t nbReadBytes;
-
- // Check Archive Header
- nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);
- if (nbReadBytes==0) return 0; // EOF
- if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");
- magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format
- if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers
-
- switch(magicNumber)
- {
- case LZ4S_MAGICNUMBER:
- return DEFAULT_DECOMPRESSOR(finput, foutput);
- case LEGACY_MAGICNUMBER:
- DISPLAYLEVEL(4, "Detected : Legacy format \n");
- return decodeLegacyStream(finput, foutput);
- case LZ4S_SKIPPABLE0:
- DISPLAYLEVEL(4, "Skipping detected skippable area \n");
- nbReadBytes = fread(&size, 1, 4, finput);
- if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
- size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format
- errorNb = fseek(finput, size, SEEK_CUR);
- if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
- return selectDecoder(finput, foutput);
- EXTENDED_FORMAT;
- default:
- if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream
- DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
- return 0;
- }
-}
-
-
-int decodeFile(char* input_filename, char* output_filename)
-{
- unsigned long long filesize = 0, decodedSize=0;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
-
-
- // Init
- start = clock();
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
-
- // Loop over multiple streams
- do
- {
- decodedSize = selectDecoder(finput, foutput);
- filesize += decodedSize;
- } while (decodedSize);
-
- // Final Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close
- fclose(finput);
- fclose(foutput);
-
- // Error status = OK
- return 0;
-}
-
-
-void waitEnter()
-{
- DISPLAY("Press enter to continue...\n");
- getchar();
-}
-
-
-int main(int argc, char** argv)
-{
- int i,
- cLevel=0,
- decode=0,
- bench=0,
- filenamesStart=2,
- legacy_format=0,
- forceStdout=0,
- forceCompress=0,
- pause=0;
- char* input_filename=0;
- char* output_filename=0;
- char nullOutput[] = NULL_OUTPUT;
- char extension[] = LZ4_EXTENSION;
-
- // Init
- programName = argv[0];
-
- for(i=1; i<argc; i++)
- {
- char* argument = argv[i];
-
- if(!argument) continue; // Protection if argument empty
-
- // Decode command (note : aggregated commands are allowed)
- if (argument[0]=='-')
- {
- // '-' means stdin/stdout
- if (argument[1]==0)
- {
- if (!input_filename) input_filename=stdinmark;
- else output_filename=stdoutmark;
- }
-
- while (argument[1]!=0)
- {
- argument ++;
-
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
- // Legacy options (-c0, -c1, -hc, -y, -s)
- if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression)
- if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression)
- if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression)
- if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission)
- if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS
-
- switch(argument[0])
- {
- // Display help
- case 'V': DISPLAY(WELCOME_MESSAGE); return 0; // Version
- case 'h': usage_advanced(); return 0;
- case 'H': usage_advanced(); usage_longhelp(); return 0;
-
- // Compression (default)
- case 'z': forceCompress = 1; break;
-
- // Compression level
- case '0':
- case '1':
- case '2':
- case '3':
- case '4':
- case '5':
- case '6':
- case '7':
- case '8':
- case '9': cLevel=*argument -'0'; break;
-
- // Use Legacy format (hidden option)
- case 'l': legacy_format=1; break;
-
- // Decoding
- case 'd': decode=1; break;
-
- // Force stdout, even if stdout==console
- case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;
-
- // Test
- case 't': decode=1; output_filename=nulmark; break;
-
- // Overwrite
- case 'f': overwrite=1; break;
-
- // Verbose mode
- case 'v': displayLevel=4; break;
-
- // Quiet mode
- case 'q': displayLevel--; break;
-
- // keep source file (default anyway, so useless) (for xz/lzma compatibility)
- case 'k': break;
-
- // Modify Block Properties
- case 'B':
- while (argument[1]!=0)
- {
- int exitBlockProperties=0;
- switch(argument[1])
- {
- case '4':
- case '5':
- case '6':
- case '7':
- {
- int B = argument[1] - '0';
- int S = 1 << (8 + 2*B);
- BMK_SetBlocksize(S);
- blockSizeId = B;
- argument++;
- break;
- }
- case 'D': blockIndependence = 0, argument++; break;
- case 'X': blockChecksum = 1, argument ++; break;
- default : exitBlockProperties=1;
- }
- if (exitBlockProperties) break;
- }
- break;
-
- // Modify Stream properties
- case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); }
-
- // Benchmark
- case 'b': bench=1; break;
-
- // Modify Nb Iterations (benchmark only)
- case 'i':
- if ((argument[1] >='1') && (argument[1] <='9'))
- {
- int iters = argument[1] - '0';
- BMK_SetNbIterations(iters);
- argument++;
- }
- break;
-
- // Pause at the end (hidden option)
- case 'p': pause=1; BMK_SetPause(); break;
-
- EXTENDED_ARGUMENTS;
-
- // Unrecognised command
- default : badusage();
- }
- }
- continue;
- }
-
- // first provided filename is input
- if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
-
- // second provided filename is output
- if (!output_filename)
- {
- output_filename=argument;
- if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
- continue;
- }
- }
-
- DISPLAYLEVEL(3, WELCOME_MESSAGE);
- DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2)));
-
- // No input filename ==> use stdin
- if(!input_filename) { input_filename=stdinmark; }
-
- // Check if input or output are defined as console; trigger an error in this case
- if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
-
- // Check if benchmark is selected
- if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);
-
- // No output filename ==> try to select one automatically (when possible)
- while (!output_filename)
- {
- if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console)
- if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension
- {
- size_t l = strlen(input_filename);
- if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;
- }
- if (!decode) // compression to file
- {
- size_t l = strlen(input_filename);
- output_filename = (char*)calloc(1,l+5);
- strcpy(output_filename, input_filename);
- strcpy(output_filename+l, LZ4_EXTENSION);
- DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
- break;
- }
- // decompression to file (automatic name will work only if input filename has format extension ".lz4")
- {
- size_t outl;
- size_t inl = strlen(input_filename);
- output_filename = (char*)calloc(1,inl+1);
- strcpy(output_filename, input_filename);
- outl = inl;
- if (inl>4)
- while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;
- if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
- DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
- }
- }
-
- // No warning message in pure pipe mode (stdin + stdout)
- if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
-
- // Check if input or output are defined as console; trigger an error in this case
- if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
- if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
-
- // Decompress input if selected
- if (decode) decodeFile(input_filename, output_filename);
- else
- // compression is default action
- {
- if (legacy_format)
- {
- DISPLAYLEVEL(2, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n");
- legacy_compress_file(input_filename, output_filename, cLevel);
- }
- else
- {
- DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
- }
- }
-
- if (pause) waitEnter();
-}
+/* + LZ4cli.c - LZ4 Command Line Interface + Copyright (C) Yann Collet 2011-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* + Note : this is stand-alone program. + It is not part of LZ4 compression library, it is a user program of the LZ4 library. + The license of LZ4 library is BSD. + The license of xxHash library is BSD. + The license of this compression CLI program is GPLv2. +*/ + +//************************************** +// Tuning parameters +//************************************** +// DISABLE_LZ4C_LEGACY_OPTIONS : +// Control the availability of -c0, -c1 and -hc legacy arguments +// Default : Legacy options are enabled +// #define DISABLE_LZ4C_LEGACY_OPTIONS + + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# define _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_DEPRECATE // VS2005 +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix +#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix + + +//**************************** +// Includes +//**************************** +#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout +#include <stdlib.h> // malloc +#include <string.h> // strcmp, strlen +#include <time.h> // clock +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" +#include "bench.h" + + +//**************************** +// OS-specific Includes +//**************************** +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include <fcntl.h> // _O_BINARY +# include <io.h> // _setmode, _isatty +# ifdef __MINGW32__ + int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h> +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include <unistd.h> // isatty +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + +//************************************** +// Compiler-specific functions +//************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if defined(_MSC_VER) // Visual Studio +# define swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define swap32 __builtin_bswap32 +#else + static inline unsigned int swap32(unsigned int x) + { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); + } +#endif + + +//**************************** +// Constants +//**************************** +#define COMPRESSOR_NAME "LZ4 Compression CLI" +#ifndef LZ4_VERSION +# define LZ4_VERSION "v1.1.0" +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ +#define LZ4_EXTENSION ".lz4" + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define MAGICNUMBER_SIZE 4 +#define LZ4S_MAGICNUMBER 0x184D2204 +#define LZ4S_SKIPPABLE0 0x184D2A50 +#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 +#define LEGACY_MAGICNUMBER 0x184C2102 + +#define CACHELINE 64 +#define LEGACY_BLOCKSIZE (8 MB) +#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) +#define LZ4S_BLOCKSIZEID_DEFAULT 7 +#define LZ4S_CHECKSUM_SEED 0 +#define LZ4S_EOS 0 +#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) + + +//************************************** +// Architecture Macros +//************************************** +static const int one = 1; +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) + + +//************************************** +// Macros +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + +//************************************** +// Special input/output +//************************************** +#define NULL_OUTPUT "null" +char stdinmark[] = "stdin"; +char stdoutmark[] = "stdout"; +#ifdef _WIN32 +char nulmark[] = "nul"; +#else +char nulmark[] = "/dev/null"; +#endif + + +//************************************** +// Local Parameters +//************************************** +static char* programName; +static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information +static int overwrite = 0; +static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; +static int blockChecksum = 0; +static int streamChecksum = 1; +static int blockIndependence = 1; + + +//************************************** +// Exceptions +//************************************** +#define DEBUG 0 +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +//************************************** +// Version modifiers +//************************************** +#define EXTENDED_ARGUMENTS +#define EXTENDED_HELP +#define EXTENDED_FORMAT +#define DEFAULT_COMPRESSOR compress_file +#define DEFAULT_DECOMPRESSOR decodeLZ4S + + +//**************************** +// Functions +//**************************** +int usage() +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] [input] [output]\n", programName); + DISPLAY( "\n"); + DISPLAY( "input : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); + DISPLAY( "Arguments :\n"); + DISPLAY( " -1 : Fast compression (default) \n"); + DISPLAY( " -9 : High compression \n"); + DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); + DISPLAY( " -z : force compression\n"); + DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + return 0; +} + +int usage_advanced() +{ + DISPLAY(WELCOME_MESSAGE); + usage(); + DISPLAY( "\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); + DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -t : test compressed file integrity\n"); + DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); + DISPLAY( " -B# : Block size [4-7](default : 7)\n"); + DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -BX : enable block checksum (default:disabled)\n"); + DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b : benchmark file(s)\n"); + DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "Legacy arguments :\n"); + DISPLAY( " -c0 : fast compression\n"); + DISPLAY( " -c1 : high compression\n"); + DISPLAY( " -hc : high compression\n"); + DISPLAY( " -y : overwrite output without prompting \n"); + DISPLAY( " -s : suppress warnings \n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + EXTENDED_HELP; + return 0; +} + +int usage_longhelp() +{ + DISPLAY( "\n"); + DISPLAY( "Which values can get [output] ? \n"); + DISPLAY( "[output] : a filename\n"); + DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); + DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT); + DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n"); + DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); + DISPLAY( " - if stdout is console : \n"); + DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION); + DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION); + DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION); + DISPLAY( "\n"); + DISPLAY( "Compression levels : \n"); + DISPLAY( "There are technically 2 accessible compression levels.\n"); + DISPLAY( "-0 ... -2 => Fast compression\n"); + DISPLAY( "-3 ... -9 => High compression\n"); + DISPLAY( "\n"); + DISPLAY( "stdin, stdout and the console : \n"); + DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); + DISPLAY( "%s will refuse to read from console, or write to console \n", programName); + DISPLAY( "except if '-c' command is specified, to force output to console \n"); + DISPLAY( "\n"); + DISPLAY( "Simple example :\n"); + DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); + DISPLAY( " %s filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments can be appended together, or provided independently. For example :\n"); + DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); + DISPLAY( " %s -f9 filename \n", programName); + DISPLAY( " is equivalent to :\n"); + DISPLAY( " %s -f -9 filename \n", programName); + DISPLAY( "\n"); + DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName); + DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); + DISPLAY( " generator | %s | consumer \n", programName); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "\n"); + DISPLAY( "Warning :\n"); + DISPLAY( "Legacy arguments take precedence. Therefore : \n"); + DISPLAY( " %s -hc filename\n", programName); + DISPLAY( "means 'compress filename in high compression mode'\n"); + DISPLAY( "It is not equivalent to :\n"); + DISPLAY( " %s -h -c filename\n", programName); + DISPLAY( "which would display help text and exit\n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + return 0; +} + +int badusage() +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (displayLevel >= 1) usage(); + exit(1); +} + + +static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } +static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } +static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } + + +int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) +{ + + if (!strcmp (input_filename, stdinmark)) + { + DISPLAYLEVEL(4,"Using stdin for input\n"); + *pfinput = stdin; + SET_BINARY_MODE(stdin); + } + else + { + *pfinput = fopen(input_filename, "rb"); + } + + if (!strcmp (output_filename, stdoutmark)) + { + DISPLAYLEVEL(4,"Using stdout for output\n"); + *pfoutput = stdout; + SET_BINARY_MODE(stdout); + } + else + { + // Check if destination file already exists + *pfoutput=0; + if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); + if (*pfoutput!=0) + { + fclose(*pfoutput); + if (!overwrite) + { + char ch; + DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); + DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); + if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible + ch = (char)getchar(); + if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); + } + } + *pfoutput = fopen( output_filename, "wb" ); + } + + if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); + if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); + + return 0; +} + + + +int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = MAGICNUMBER_SIZE; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int displayLevel = (compressionlevel>0); + clock_t start, end; + size_t sizeCheck; + + + // Init + if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + + // Allocate Memory + in_buff = (char*)malloc(LEGACY_BLOCKSIZE); + out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); + sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); + + // Main Loop + while (1) + { + unsigned int outSize; + // Read Block + int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + if( inSize<=0 ) break; + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, inSize); + compressedfilesize += outSize+4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) +{ + void* (*initFunction) (const char*); + int (*compressionFunction)(void*, const char*, char*, int, int); + char* (*translateFunction) (void*); + int (*freeFunction) (void*); + void* ctx; + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff, *in_start, *in_end; + char* out_buff; + FILE* finput; + FILE* foutput; + clock_t start, end; + unsigned int blockSize, inputBufferSize; + size_t sizeCheck, header_size; + void* streamChecksumState=NULL; + + + // Init + start = clock(); + if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; + if (compressionlevel>=3) + { + initFunction = LZ4_createHC; + compressionFunction = LZ4_compressHC_limitedOutput_continue; + translateFunction = LZ4_slideInputBufferHC; + freeFunction = LZ4_freeHC; + } + else + { + initFunction = LZ4_create; + compressionFunction = LZ4_compress_limitedOutput_continue; + translateFunction = LZ4_slideInputBuffer; + freeFunction = LZ4_free; + } + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + inputBufferSize = blockSize + 64 KB; + if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE; + in_buff = (char*)malloc(inputBufferSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); + in_start = in_buff; in_end = in_buff + inputBufferSize; + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + ctx = initFunction(in_buff); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') + *(out_buff+4) |= (blockIndependence & _1BIT) << 5; + *(out_buff+4) |= (blockChecksum & _1BIT) << 4; + *(out_buff+4) |= (streamChecksum & _1BIT) << 2; + *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(out_buff+6) = (unsigned char) checkbits; + header_size = 7; + sizeCheck = fwrite(out_buff, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // Main Loop + while (1) + { + unsigned int outSize; + unsigned int inSize; + // Read Block + if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); + inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); + if( inSize==0 ) break; // No more input : end of compression + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); + + // Compress Block + outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + + } + else // Copy Original + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_start, 1, inSize, foutput); + if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + in_start += inSize; + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + freeFunction(ctx); + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int compress_file(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff; + char* out_buff; + char* headerBuffer; + FILE* finput; + FILE* foutput; + clock_t start, end; + int blockSize; + size_t sizeCheck, header_size, readSize; + void* streamChecksumState=NULL; + + // Branch out + if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel); + + // Init + start = clock(); + if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; + if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput; + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + in_buff = (char*)malloc(blockSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); + if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Write Archive Header + *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') + *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; + *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; + *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; + *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(headerBuffer+6) = (unsigned char) checkbits; + header_size = 7; + + // Write header + sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // read first block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + + // Main Loop + while (readSize>0) + { + unsigned int outSize; + + filesize += readSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + } + else // Copy Original Uncompressed + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_buff, 1, readSize, foutput); + if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + + // Read next block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Close & Free + free(in_buff); + free(out_buff); + free(headerBuffer); + fclose(finput); + fclose(foutput); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + return 0; +} + + +unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff; + unsigned int blockSize; + + + // Allocate Memory + in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + out_buff = (char*)malloc(LEGACY_BLOCKSIZE); + if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); + + // Main Loop + while (1) + { + int decodeSize; + size_t sizeCheck; + + // Block Size + sizeCheck = fread(&blockSize, 1, 4, finput); + if (sizeCheck==0) break; // Nothing to read : file read is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian + if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) + { // Cannot read next block : maybe new stream ? + fseek(finput, -4, SEEK_CUR); + break; + } + + // Read Block + sizeCheck = fread(in_buff, 1, blockSize, finput); + + // Decode Block + decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); + filesize += decodeSize; + + // Write Block + sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); + if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff, *out_start, *out_end; + unsigned char descriptor[LZ4S_MAXHEADERSIZE]; + size_t nbReadBytes; + int decodedBytes=0; + unsigned int maxBlockSize; + size_t sizeCheck; + int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; + void* streamChecksumState=NULL; + int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; + unsigned int prefix64k = 0; + + // Decode stream descriptor + nbReadBytes = fread(descriptor, 1, 3, finput); + if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); + { + int version = (descriptor[0] >> 6) & _2BITS; + int streamSize = (descriptor[0] >> 3) & _1BIT; + int reserved1 = (descriptor[0] >> 1) & _1BIT; + int dictionary = (descriptor[0] >> 0) & _1BIT; + + int reserved2 = (descriptor[1] >> 7) & _1BIT; + int blockSizeId = (descriptor[1] >> 4) & _3BITS; + int reserved3 = (descriptor[1] >> 0) & _4BITS; + int checkBits = (descriptor[2] >> 0) & _8BITS; + int checkBits_xxh32; + + blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; + blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; + streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; + + if (version != 1) EXM_THROW(62, "Wrong version number"); + if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); + if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); + if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); + if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); + if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); + // Checkbits verification + descriptor[1] &= 0xF0; + checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); + checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); + if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); + } + + if (!blockIndependenceFlag) + { + decompressionFunction = LZ4_decompress_safe_withPrefix64k; + prefix64k = 64 KB; + } + + // Allocate Memory + { + unsigned int outbuffSize = prefix64k+maxBlockSize; + in_buff = (char*)malloc(maxBlockSize); + if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; + out_buff = (char*)malloc(outbuffSize); + out_end = out_buff + outbuffSize; + out_start = out_buff + prefix64k; + if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); + } + if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Main Loop + while (1) + { + unsigned int blockSize, uncompressedFlag; + + // Block Size + nbReadBytes = fread(&blockSize, 1, 4, finput); + if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); + if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian + uncompressedFlag = blockSize >> 31; + blockSize &= 0x7FFFFFFF; + if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); + + // Read Block + nbReadBytes = fread(in_buff, 1, blockSize, finput); + if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); + + // Check Block + if (blockChecksumFlag) + { + unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); + } + + if (uncompressedFlag) + { + // Write uncompressed Block + sizeCheck = fwrite(in_buff, 1, blockSize, foutput); + if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); + filesize += blockSize; + if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); + if (!blockIndependenceFlag) + { + if (blockSize >= prefix64k) + { + memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks + out_start = out_buff + prefix64k; + continue; + } + else + { + memcpy(out_start, in_buff, blockSize); + decodedBytes = blockSize; + } + } + } + else + { + // Decode Block + decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); + if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); + filesize += decodedBytes; + if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); + + // Write Block + sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); + if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); + } + + if (!blockIndependenceFlag) + { + out_start += decodedBytes; + if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) + { + memcpy(out_buff, out_start - prefix64k, prefix64k); + out_start = out_buff + prefix64k; + } + } + } + + // Stream Checksum + if (streamChecksumFlag) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +unsigned long long selectDecoder( FILE* finput, FILE* foutput) +{ + unsigned int magicNumber, size; + int errorNb; + size_t nbReadBytes; + + // Check Archive Header + nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) return 0; // EOF + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); + magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format + if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers + + switch(magicNumber) + { + case LZ4S_MAGICNUMBER: + return DEFAULT_DECOMPRESSOR(finput, foutput); + case LEGACY_MAGICNUMBER: + DISPLAYLEVEL(4, "Detected : Legacy format \n"); + return decodeLegacyStream(finput, foutput); + case LZ4S_SKIPPABLE0: + DISPLAYLEVEL(4, "Skipping detected skippable area \n"); + nbReadBytes = fread(&size, 1, 4, finput); + if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); + size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format + errorNb = fseek(finput, size, SEEK_CUR); + if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); + return selectDecoder(finput, foutput); + EXTENDED_FORMAT; + default: + if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream + DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); + return 0; + } +} + + +int decodeFile(char* input_filename, char* output_filename) +{ + unsigned long long filesize = 0, decodedSize=0; + FILE* finput; + FILE* foutput; + clock_t start, end; + + + // Init + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + + // Loop over multiple streams + do + { + decodedSize = selectDecoder(finput, foutput); + filesize += decodedSize; + } while (decodedSize); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close + fclose(finput); + fclose(foutput); + + // Error status = OK + return 0; +} + + +void waitEnter() +{ + DISPLAY("Press enter to continue...\n"); + getchar(); +} + + +int main(int argc, char** argv) +{ + int i, + cLevel=0, + decode=0, + bench=0, + filenamesStart=2, + legacy_format=0, + forceStdout=0, + forceCompress=0, + pause=0; + char* input_filename=0; + char* output_filename=0; + char* dynNameSpace=0; + char nullOutput[] = NULL_OUTPUT; + char extension[] = LZ4_EXTENSION; + + // Init + programName = argv[0]; + + for(i=1; i<argc; i++) + { + char* argument = argv[i]; + + if(!argument) continue; // Protection if argument empty + + // Decode command (note : aggregated commands are allowed) + if (argument[0]=='-') + { + // '-' means stdin/stdout + if (argument[1]==0) + { + if (!input_filename) input_filename=stdinmark; + else output_filename=stdoutmark; + } + + while (argument[1]!=0) + { + argument ++; + +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + // Legacy options (-c0, -c1, -hc, -y, -s) + if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression) + if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression) + if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression) + if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission) + if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode) +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + + switch(argument[0]) + { + // Display help + case 'V': DISPLAY(WELCOME_MESSAGE); return 0; // Version + case 'h': usage_advanced(); return 0; + case 'H': usage_advanced(); usage_longhelp(); return 0; + + // Compression (default) + case 'z': forceCompress = 1; break; + + // Compression level + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': cLevel=*argument -'0'; break; + + // Use Legacy format (for Linux kernel compression) + case 'l': legacy_format=1; break; + + // Decoding + case 'd': decode=1; break; + + // Force stdout, even if stdout==console + case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break; + + // Test + case 't': decode=1; output_filename=nulmark; break; + + // Overwrite + case 'f': overwrite=1; break; + + // Verbose mode + case 'v': displayLevel=4; break; + + // Quiet mode + case 'q': displayLevel--; break; + + // keep source file (default anyway, so useless) (for xz/lzma compatibility) + case 'k': break; + + // Modify Block Properties + case 'B': + while (argument[1]!=0) + { + int exitBlockProperties=0; + switch(argument[1]) + { + case '4': + case '5': + case '6': + case '7': + { + int B = argument[1] - '0'; + int S = 1 << (8 + 2*B); + BMK_SetBlocksize(S); + blockSizeId = B; + argument++; + break; + } + case 'D': blockIndependence = 0, argument++; break; + case 'X': blockChecksum = 1, argument ++; break; + default : exitBlockProperties=1; + } + if (exitBlockProperties) break; + } + break; + + // Modify Stream properties + case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); } + + // Benchmark + case 'b': bench=1; break; + + // Modify Nb Iterations (benchmark only) + case 'i': + if ((argument[1] >='1') && (argument[1] <='9')) + { + int iters = argument[1] - '0'; + BMK_SetNbIterations(iters); + argument++; + } + break; + + // Pause at the end (hidden option) + case 'p': pause=1; BMK_SetPause(); break; + + EXTENDED_ARGUMENTS; + + // Unrecognised command + default : badusage(); + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + // second provided filename is output + if (!output_filename) + { + output_filename=argument; + if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; + continue; + } + } + + DISPLAYLEVEL(3, WELCOME_MESSAGE); + DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2))); + + // No input filename ==> use stdin + if(!input_filename) { input_filename=stdinmark; } + + // Check if input or output are defined as console; trigger an error in this case + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + + // Check if benchmark is selected + if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + + // No output filename ==> try to select one automatically (when possible) + while (!output_filename) + { + if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console) + if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension + { + size_t l = strlen(input_filename); + if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; + } + if (!decode) // compression to file + { + size_t l = strlen(input_filename); + dynNameSpace = (char*)calloc(1,l+5); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + strcpy(output_filename+l, LZ4_EXTENSION); + DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); + break; + } + // decompression to file (automatic name will work only if input filename has correct format extension) + { + size_t outl; + size_t inl = strlen(input_filename); + dynNameSpace = (char*)calloc(1,inl+1); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + outl = inl; + if (inl>4) + while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } + DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); + } + } + + // No warning message in pure pipe mode (stdin + stdout) + if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + + // Check if input or output are defined as console; trigger an error in this case + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); + + // Decompress input if selected + if (decode) decodeFile(input_filename, output_filename); + else + // compression is default action + { + if (legacy_format) + { + DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); + legacy_compress_file(input_filename, output_filename, cLevel); + } + else + { + DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + } + } + + if (pause) waitEnter(); + free(dynNameSpace); + return 0; +} @@ -1,817 +1,856 @@ -/*
- LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-//**************************************
-// Memory routines
-//**************************************
-#include <stdlib.h> // calloc, free
-#define ALLOCATOR(s) calloc(1,s)
-#define FREEMEM free
-#include <string.h> // memset, memcpy
-#define MEM_INIT memset
-
-
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
-#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
-# define LZ4_ARCH64 1
-#else
-# define LZ4_ARCH64 0
-#endif
-
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
-#if defined (__GLIBC__)
-# include <endian.h>
-# if (__BYTE_ORDER == __BIG_ENDIAN)
-# define LZ4_BIG_ENDIAN 1
-# endif
-#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))
-# define LZ4_BIG_ENDIAN 1
-#elif defined(__sparc) || defined(__sparc__) \
- || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
- || defined(__hpux) || defined(__hppa) \
- || defined(_MIPSEB) || defined(__s390__)
-# define LZ4_BIG_ENDIAN 1
-#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
-#endif
-
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
-#if defined(__ARM_FEATURE_UNALIGNED)
-# define LZ4_FORCE_UNALIGNED_ACCESS 1
-#endif
-
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
-# define LZ4_FORCE_SW_BITCOUNT
-#endif
-
-
-//**************************************
-// Compiler Options
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
- /* "restrict" is a known keyword */
-#else
-# define restrict // Disable restrict
-#endif
-
-#ifdef _MSC_VER // Visual Studio
-# define FORCE_INLINE static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
-# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-#endif
-
-#ifdef _MSC_VER // Visual Studio
-# define lz4_bswap16(x) _byteswap_ushort(x)
-#else
-# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
-#endif
-
-
-//**************************************
-// Includes
-//**************************************
-#include "lz4hc.h"
-#include "lz4.h"
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
-# define _PACKED __attribute__ ((packed))
-#else
-# define _PACKED
-#endif
-
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# ifdef __IBMC__
-# pragma pack(1)
-# else
-# pragma pack(push, 1)
-# endif
-#endif
-
-typedef struct _U16_S { U16 v; } _PACKED U16_S;
-typedef struct _U32_S { U32 v; } _PACKED U32_S;
-typedef struct _U64_S { U64 v; } _PACKED U64_S;
-
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# pragma pack(pop)
-#endif
-
-#define A64(x) (((U64_S *)(x))->v)
-#define A32(x) (((U32_S *)(x))->v)
-#define A16(x) (((U16_S *)(x))->v)
-
-
-//**************************************
-// Constants
-//**************************************
-#define MINMATCH 4
-
-#define DICTIONARY_LOGSIZE 16
-#define MAXD (1<<DICTIONARY_LOGSIZE)
-#define MAXD_MASK ((U32)(MAXD - 1))
-#define MAX_DISTANCE (MAXD - 1)
-
-#define HASH_LOG (DICTIONARY_LOGSIZE-1)
-#define HASHTABLESIZE (1 << HASH_LOG)
-#define HASH_MASK (HASHTABLESIZE - 1)
-
-#define MAX_NB_ATTEMPTS 256
-
-#define ML_BITS 4
-#define ML_MASK (size_t)((1U<<ML_BITS)-1)
-#define RUN_BITS (8-ML_BITS)
-#define RUN_MASK ((1U<<RUN_BITS)-1)
-
-#define COPYLENGTH 8
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH+MINMATCH)
-#define MINLENGTH (MFLIMIT+1)
-#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-
-//**************************************
-// Architecture-specific macros
-//**************************************
-#if LZ4_ARCH64 // 64-bit
-# define STEPSIZE 8
-# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
-# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
-# define UARCH U64
-# define AARCH A64
-# define HTYPE U32
-# define INITBASE(b,s) const BYTE* const b = s
-#else // 32-bit
-# define STEPSIZE 4
-# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
-# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
-# define UARCH U32
-# define AARCH A32
-//# define HTYPE const BYTE*
-//# define INITBASE(b,s) const int b = 0
-# define HTYPE U32
-# define INITBASE(b,s) const BYTE* const b = s
-#endif
-
-#if defined(LZ4_BIG_ENDIAN)
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
-# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
-# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
-#endif
-
-
-//************************************************************
-// Local Types
-//************************************************************
-typedef struct
-{
- const BYTE* inputBuffer;
- const BYTE* base;
- const BYTE* end;
- HTYPE hashTable[HASHTABLESIZE];
- U16 chainTable[MAXD];
- const BYTE* nextToUpdate;
-} LZ4HC_Data_Structure;
-
-
-//**************************************
-// Macros
-//**************************************
-#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
-#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
-#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
-#define HASH_VALUE(p) HASH_FUNCTION(A32(p))
-#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base)
-#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
-#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
-
-
-//**************************************
-// Private functions
-//**************************************
-#if LZ4_ARCH64
-
-FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
-{
-#if defined(LZ4_BIG_ENDIAN)
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
-# else
- int r;
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
-# endif
-#else
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanForward64( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll(val) >> 3);
-# else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58];
-# endif
-#endif
-}
-
-#else
-
-FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
-{
-#if defined(LZ4_BIG_ENDIAN)
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r;
- _BitScanReverse( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz(val) >> 3);
-# else
- int r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
-# endif
-#else
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r;
- _BitScanForward( &r, val );
- return (int)(r>>3);
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz(val) >> 3);
-# else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
-#endif
-}
-
-#endif
-
-
-FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
-{
- MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
- MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
- hc4->nextToUpdate = base + 1;
- hc4->base = base;
- hc4->inputBuffer = base;
- hc4->end = base;
-}
-
-
-void* LZ4_createHC (const char* inputBuffer)
-{
- void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
- LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
- return hc4;
-}
-
-
-int LZ4_freeHC (void* LZ4HC_Data)
-{
- FREEMEM(LZ4HC_Data);
- return (0);
-}
-
-
-// Update chains up to ip (excluded)
-FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
-{
- U16* chainTable = hc4->chainTable;
- HTYPE* HashTable = hc4->hashTable;
- INITBASE(base,hc4->base);
-
- while(hc4->nextToUpdate < ip)
- {
- const BYTE* const p = hc4->nextToUpdate;
- size_t delta = (p) - HASH_POINTER(p);
- if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
- DELTANEXT(p) = (U16)delta;
- HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);
- hc4->nextToUpdate++;
- }
-}
-
-
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
-{
- LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
- U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;
- distance = (distance >> 16) << 16; // Must be a multiple of 64 KB
- LZ4HC_Insert(hc4, hc4->end - MINMATCH);
- memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
- hc4->nextToUpdate -= distance;
- hc4->base -= distance;
- if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow
- {
- int i;
- hc4->base += 1 GB;
- for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
- }
- hc4->end -= distance;
- return (char*)(hc4->end);
-}
-
-
-FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
-{
- const BYTE* p1t = p1;
-
- while (p1t<matchlimit-(STEPSIZE-1))
- {
- UARCH diff = AARCH(p2) ^ AARCH(p1t);
- if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }
- p1t += LZ4_NbCommonBytes(diff);
- return (p1t - p1);
- }
- if (LZ4_ARCH64) if ((p1t<(matchlimit-3)) && (A32(p2) == A32(p1t))) { p1t+=4; p2+=4; }
- if ((p1t<(matchlimit-1)) && (A16(p2) == A16(p1t))) { p1t+=2; p2+=2; }
- if ((p1t<matchlimit) && (*p2 == *p1t)) p1t++;
- return (p1t - p1);
-}
-
-
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)
-{
- U16* const chainTable = hc4->chainTable;
- HTYPE* const HashTable = hc4->hashTable;
- const BYTE* ref;
- INITBASE(base,hc4->base);
- int nbAttempts=MAX_NB_ATTEMPTS;
- size_t repl=0, ml=0;
- U16 delta=0; // useless assignment, to remove an uninitialization warning
-
- // HC4 match finder
- LZ4HC_Insert(hc4, ip);
- ref = HASH_POINTER(ip);
-
-#define REPEAT_OPTIMIZATION
-#ifdef REPEAT_OPTIMIZATION
- // Detect repetitive sequences of length <= 4
- if ((U32)(ip-ref) <= 4) // potential repetition
- {
- if (A32(ref) == A32(ip)) // confirmed
- {
- delta = (U16)(ip-ref);
- repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
- *matchpos = ref;
- }
- ref = GETNEXT(ref);
- }
-#endif
-
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
- {
- nbAttempts--;
- if (*(ref+ml) == *(ip+ml))
- if (A32(ref) == A32(ip))
- {
- size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
- if (mlt > ml) { ml = mlt; *matchpos = ref; }
- }
- ref = GETNEXT(ref);
- }
-
-#ifdef REPEAT_OPTIMIZATION
- // Complete table
- if (repl)
- {
- const BYTE* ptr = ip;
- const BYTE* end;
-
- end = ip + repl - (MINMATCH-1);
- while(ptr < end-delta)
- {
- DELTANEXT(ptr) = delta; // Pre-Load
- ptr++;
- }
- do
- {
- DELTANEXT(ptr) = delta;
- HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain
- ptr++;
- } while(ptr < end);
- hc4->nextToUpdate = end;
- }
-#endif
-
- return (int)ml;
-}
-
-
-FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)
-{
- U16* const chainTable = hc4->chainTable;
- HTYPE* const HashTable = hc4->hashTable;
- INITBASE(base,hc4->base);
- const BYTE* ref;
- int nbAttempts = MAX_NB_ATTEMPTS;
- int delta = (int)(ip-startLimit);
-
- // First Match
- LZ4HC_Insert(hc4, ip);
- ref = HASH_POINTER(ip);
-
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
- {
- nbAttempts--;
- if (*(startLimit + longest) == *(ref - delta + longest))
- if (A32(ref) == A32(ip))
- {
-#if 1
- const BYTE* reft = ref+MINMATCH;
- const BYTE* ipt = ip+MINMATCH;
- const BYTE* startt = ip;
-
- while (ipt<matchlimit-(STEPSIZE-1))
- {
- UARCH diff = AARCH(reft) ^ AARCH(ipt);
- if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
- ipt += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; }
- if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; }
- if ((ipt<matchlimit) && (*reft == *ipt)) ipt++;
-_endCount:
- reft = ref;
-#else
- // Easier for code maintenance, but unfortunately slower too
- const BYTE* startt = ip;
- const BYTE* reft = ref;
- const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
-#endif
-
- while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}
-
- if ((ipt-startt) > longest)
- {
- longest = (int)(ipt-startt);
- *matchpos = reft;
- *startpos = startt;
- }
- }
- ref = GETNEXT(ref);
- }
-
- return longest;
-}
-
-
-typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
-
-FORCE_INLINE int LZ4HC_encodeSequence (
- const BYTE** ip,
- BYTE** op,
- const BYTE** anchor,
- int matchLength,
- const BYTE* ref,
- limitedOutput_directive limitedOutputBuffer,
- BYTE* oend)
-{
- int length;
- BYTE* token;
-
- // Encode Literal length
- length = (int)(*ip - *anchor);
- token = (*op)++;
- if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit
- if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
- else *token = (BYTE)(length<<ML_BITS);
-
- // Copy Literals
- LZ4_BLINDCOPY(*anchor, *op, length);
-
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
-
- // Encode MatchLength
- length = (int)(matchLength-MINMATCH);
- if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit
- if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
- else *token += (BYTE)(length);
-
- // Prepare next loop
- *ip += matchLength;
- *anchor = *ip;
-
- return 0;
-}
-
-
-static int LZ4HC_compress_generic (
- void* ctxvoid,
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize,
- limitedOutput_directive limit
- )
-{
- LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
- const BYTE* ip = (const BYTE*) source;
- const BYTE* anchor = ip;
- const BYTE* const iend = ip + inputSize;
- const BYTE* const mflimit = iend - MFLIMIT;
- const BYTE* const matchlimit = (iend - LASTLITERALS);
-
- BYTE* op = (BYTE*) dest;
- BYTE* const oend = op + maxOutputSize;
-
- int ml, ml2, ml3, ml0;
- const BYTE* ref=NULL;
- const BYTE* start2=NULL;
- const BYTE* ref2=NULL;
- const BYTE* start3=NULL;
- const BYTE* ref3=NULL;
- const BYTE* start0;
- const BYTE* ref0;
-
-
- // Ensure blocks follow each other
- if (ip != ctx->end) return 0;
- ctx->end += inputSize;
-
- ip++;
-
- // Main Loop
- while (ip < mflimit)
- {
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));
- if (!ml) { ip++; continue; }
-
- // saved, in case we would skip too much
- start0 = ip;
- ref0 = ref;
- ml0 = ml;
-
-_Search2:
- if (ip+ml < mflimit)
- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
- else ml2 = ml;
-
- if (ml2 == ml) // No better match
- {
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
- continue;
- }
-
- if (start0 < ip)
- {
- if (start2 < ip + ml0) // empirical
- {
- ip = start0;
- ref = ref0;
- ml = ml0;
- }
- }
-
- // Here, start0==ip
- if ((start2 - ip) < 3) // First Match too small : removed
- {
- ml = ml2;
- ip = start2;
- ref =ref2;
- goto _Search2;
- }
-
-_Search3:
- // Currently we have :
- // ml2 > ml1, and
- // ip1+3 <= ip2 (usually < ip1+ml1)
- if ((start2 - ip) < OPTIMAL_ML)
- {
- int correction;
- int new_ml = ml;
- if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
- if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
- correction = new_ml - (int)(start2 - ip);
- if (correction > 0)
- {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- }
- // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18)
-
- if (start2 + ml2 < mflimit)
- ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
- else ml3 = ml2;
-
- if (ml3 == ml2) // No better match : 2 sequences to encode
- {
- // ip & ref are known; Now for ml
- if (start2 < ip+ml) ml = (int)(start2 - ip);
- // Now, encode 2 sequences
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
- ip = start2;
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
- continue;
- }
-
- if (start3 < ip+ml+3) // Not enough space for match 2 : remove it
- {
- if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
- {
- if (start2 < ip+ml)
- {
- int correction = (int)(ip+ml - start2);
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- if (ml2 < MINMATCH)
- {
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- }
- }
-
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
- ip = start3;
- ref = ref3;
- ml = ml3;
-
- start0 = start2;
- ref0 = ref2;
- ml0 = ml2;
- goto _Search2;
- }
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- goto _Search3;
- }
-
- // OK, now we have 3 ascending matches; let's write at least the first one
- // ip & ref are known; Now for ml
- if (start2 < ip+ml)
- {
- if ((start2 - ip) < (int)ML_MASK)
- {
- int correction;
- if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
- if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
- correction = ml - (int)(start2 - ip);
- if (correction > 0)
- {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- }
- else
- {
- ml = (int)(start2 - ip);
- }
- }
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
-
- ip = start2;
- ref = ref2;
- ml = ml2;
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
-
- goto _Search3;
-
- }
-
- // Encode Last Literals
- {
- int lastRun = (int)(iend - anchor);
- if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (BYTE)(lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-int LZ4_compressHC(const char* source, char* dest, int inputSize)
-{
- void* ctx = LZ4_createHC(source);
- int result;
- if (ctx==NULL) return 0;
-
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit);
-
- LZ4_freeHC(ctx);
- return result;
-}
-
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
-{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit);
-}
-
-
-int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- void* ctx = LZ4_createHC(source);
- int result;
- if (ctx==NULL) return 0;
-
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput);
-
- LZ4_freeHC(ctx);
- return result;
-}
-
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput);
-}
-
+/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Memory routines +//************************************** +#include <stdlib.h> // calloc, free +#define ALLOCATOR(s) calloc(1,s) +#define FREEMEM free +#include <string.h> // memset, memcpy +#define MEM_INIT memset + + +//************************************** +// CPU Feature Detection +//************************************** +// 32 or 64 bits ? +#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ + || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ + || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode +# define LZ4_ARCH64 1 +#else +# define LZ4_ARCH64 0 +#endif + +// Little Endian or Big Endian ? +// Overwrite the #define below if you know your architecture endianess +#if defined (__GLIBC__) +# include <endian.h> +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define LZ4_BIG_ENDIAN 1 +# endif +#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) +# define LZ4_BIG_ENDIAN 1 +#elif defined(__sparc) || defined(__sparc__) \ + || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) +# define LZ4_BIG_ENDIAN 1 +#else +// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +#endif + +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +#if defined(__ARM_FEATURE_UNALIGNED) +# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#endif + +// Define this parameter if your target system or compiler does not support hardware bit count +#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +//************************************** +// Compiler Options +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 + /* "restrict" is a known keyword */ +#else +# define restrict // Disable restrict +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +# include <intrin.h> // For Visual 2005 +# if LZ4_ARCH64 // 64-bits +# pragma intrinsic(_BitScanForward64) // For Visual 2005 +# pragma intrinsic(_BitScanReverse64) // For Visual 2005 +# else // 32-bits +# pragma intrinsic(_BitScanForward) // For Visual 2005 +# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# endif +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +#ifdef _MSC_VER // Visual Studio +# define lz4_bswap16(x) _byteswap_ushort(x) +#else +# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +#endif + + +//************************************** +// Includes +//************************************** +#include "lz4hc.h" +#include "lz4.h" + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U16_S { U16 v; } _PACKED U16_S; +typedef struct _U32_S { U32 v; } _PACKED U32_S; +typedef struct _U64_S { U64 v; } _PACKED U64_S; + +#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A64(x) (((U64_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A16(x) (((U16_S *)(x))->v) + + +//************************************** +// Constants +//************************************** +#define MINMATCH 4 + +#define DICTIONARY_LOGSIZE 16 +#define MAXD (1<<DICTIONARY_LOGSIZE) +#define MAXD_MASK ((U32)(MAXD - 1)) +#define MAX_DISTANCE (MAXD - 1) + +#define HASH_LOG (DICTIONARY_LOGSIZE-1) +#define HASHTABLESIZE (1 << HASH_LOG) +#define HASH_MASK (HASHTABLESIZE - 1) + +#define MAX_NB_ATTEMPTS 256 + +#define ML_BITS 4 +#define ML_MASK (size_t)((1U<<ML_BITS)-1) +#define RUN_BITS (8-ML_BITS) +#define RUN_MASK ((1U<<RUN_BITS)-1) + +#define COPYLENGTH 8 +#define LASTLITERALS 5 +#define MFLIMIT (COPYLENGTH+MINMATCH) +#define MINLENGTH (MFLIMIT+1) +#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + + +//************************************** +// Architecture-specific macros +//************************************** +#if LZ4_ARCH64 // 64-bit +# define STEPSIZE 8 +# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8; +# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d) +# define UARCH U64 +# define AARCH A64 +# define HTYPE U32 +# define INITBASE(b,s) const BYTE* const b = s +#else // 32-bit +# define STEPSIZE 4 +# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4; +# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d); +# define UARCH U32 +# define AARCH A32 +//# define HTYPE const BYTE* +//# define INITBASE(b,s) const int b = 0 +# define HTYPE U32 +# define INITBASE(b,s) const BYTE* const b = s +#endif + +#if defined(LZ4_BIG_ENDIAN) +# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } +# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; } +#else // Little Endian +# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); } +# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } +#endif + + +//************************************************************ +// Local Types +//************************************************************ +typedef struct +{ + const BYTE* inputBuffer; + const BYTE* base; + const BYTE* end; + HTYPE hashTable[HASHTABLESIZE]; + U16 chainTable[MAXD]; + const BYTE* nextToUpdate; +} LZ4HC_Data_Structure; + + +//************************************** +// Macros +//************************************** +#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e); +#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; } +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) +#define HASH_VALUE(p) HASH_FUNCTION(A32(p)) +#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) +#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] +#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) + + +//************************************** +// Private functions +//************************************** +#if LZ4_ARCH64 + +FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) +{ +#if defined(LZ4_BIG_ENDIAN) +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); +# else + int r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif +#else +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll(val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; +# endif +#endif +} + +#else + +FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) +{ +#if defined(LZ4_BIG_ENDIAN) +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanReverse( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); +# else + int r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif +#else +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, val ); + return (int)(r>>3); +# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz(val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif +#endif +} + +#endif + + +int LZ4_sizeofStreamStateHC() +{ + return sizeof(LZ4HC_Data_Structure); +} + +FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base) +{ + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); + hc4->nextToUpdate = base + 1; + hc4->base = base; + hc4->inputBuffer = base; + hc4->end = base; +} + +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) +{ + if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; // Error : pointer is not aligned for pointer (32 or 64 bits) + LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); + return 0; +} + + +void* LZ4_createHC (const char* inputBuffer) +{ + void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); + LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); + return hc4; +} + + +int LZ4_freeHC (void* LZ4HC_Data) +{ + FREEMEM(LZ4HC_Data); + return (0); +} + + +// Update chains up to ip (excluded) +FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +{ + U16* chainTable = hc4->chainTable; + HTYPE* HashTable = hc4->hashTable; + INITBASE(base,hc4->base); + + while(hc4->nextToUpdate < ip) + { + const BYTE* const p = hc4->nextToUpdate; + size_t delta = (p) - HASH_POINTER(p); + if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; + DELTANEXT(p) = (U16)delta; + HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base); + hc4->nextToUpdate++; + } +} + + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; + U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB; + distance = (distance >> 16) << 16; // Must be a multiple of 64 KB + LZ4HC_Insert(hc4, hc4->end - MINMATCH); + memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); + hc4->nextToUpdate -= distance; + hc4->base -= distance; + if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow + { + int i; + hc4->base += 1 GB; + for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB; + } + hc4->end -= distance; + return (char*)(hc4->end); +} + + +FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) +{ + const BYTE* p1t = p1; + + while (p1t<matchlimit-(STEPSIZE-1)) + { + UARCH diff = AARCH(p2) ^ AARCH(p1t); + if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; } + p1t += LZ4_NbCommonBytes(diff); + return (p1t - p1); + } + if (LZ4_ARCH64) if ((p1t<(matchlimit-3)) && (A32(p2) == A32(p1t))) { p1t+=4; p2+=4; } + if ((p1t<(matchlimit-1)) && (A16(p2) == A16(p1t))) { p1t+=2; p2+=2; } + if ((p1t<matchlimit) && (*p2 == *p1t)) p1t++; + return (p1t - p1); +} + + +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos) +{ + U16* const chainTable = hc4->chainTable; + HTYPE* const HashTable = hc4->hashTable; + const BYTE* ref; + INITBASE(base,hc4->base); + int nbAttempts=MAX_NB_ATTEMPTS; + size_t repl=0, ml=0; + U16 delta=0; // useless assignment, to remove an uninitialization warning + + // HC4 match finder + LZ4HC_Insert(hc4, ip); + ref = HASH_POINTER(ip); + +#define REPEAT_OPTIMIZATION +#ifdef REPEAT_OPTIMIZATION + // Detect repetitive sequences of length <= 4 + if ((U32)(ip-ref) <= 4) // potential repetition + { + if (A32(ref) == A32(ip)) // confirmed + { + delta = (U16)(ip-ref); + repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; + *matchpos = ref; + } + ref = GETNEXT(ref); + } +#endif + + while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) + { + nbAttempts--; + if (*(ref+ml) == *(ip+ml)) + if (A32(ref) == A32(ip)) + { + size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = ref; } + } + ref = GETNEXT(ref); + } + +#ifdef REPEAT_OPTIMIZATION + // Complete table + if (repl) + { + const BYTE* ptr = ip; + const BYTE* end; + + end = ip + repl - (MINMATCH-1); + while(ptr < end-delta) + { + DELTANEXT(ptr) = delta; // Pre-Load + ptr++; + } + do + { + DELTANEXT(ptr) = delta; + HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain + ptr++; + } while(ptr < end); + hc4->nextToUpdate = end; + } +#endif + + return (int)ml; +} + + +FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos) +{ + U16* const chainTable = hc4->chainTable; + HTYPE* const HashTable = hc4->hashTable; + INITBASE(base,hc4->base); + const BYTE* ref; + int nbAttempts = MAX_NB_ATTEMPTS; + int delta = (int)(ip-startLimit); + + // First Match + LZ4HC_Insert(hc4, ip); + ref = HASH_POINTER(ip); + + while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) + { + nbAttempts--; + if (*(startLimit + longest) == *(ref - delta + longest)) + if (A32(ref) == A32(ip)) + { +#if 1 + const BYTE* reft = ref+MINMATCH; + const BYTE* ipt = ip+MINMATCH; + const BYTE* startt = ip; + + while (ipt<matchlimit-(STEPSIZE-1)) + { + UARCH diff = AARCH(reft) ^ AARCH(ipt); + if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; } + ipt += LZ4_NbCommonBytes(diff); + goto _endCount; + } + if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; } + if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; } + if ((ipt<matchlimit) && (*reft == *ipt)) ipt++; +_endCount: + reft = ref; +#else + // Easier for code maintenance, but unfortunately slower too + const BYTE* startt = ip; + const BYTE* reft = ref; + const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit); +#endif + + while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;} + + if ((ipt-startt) > longest) + { + longest = (int)(ipt-startt); + *matchpos = reft; + *startpos = startt; + } + } + ref = GETNEXT(ref); + } + + return longest; +} + + +typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; + +FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* ref, + limitedOutput_directive limitedOutputBuffer, + BYTE* oend) +{ + int length; + BYTE* token; + + // Encode Literal length + length = (int)(*ip - *anchor); + token = (*op)++; + if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit + if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (BYTE)(length<<ML_BITS); + + // Copy Literals + LZ4_BLINDCOPY(*anchor, *op, length); + + // Encode Offset + LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref)); + + // Encode MatchLength + length = (int)(matchLength-MINMATCH); + if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit + if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } + else *token += (BYTE)(length); + + // Prepare next loop + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +static int LZ4HC_compress_generic ( + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive limit + ) +{ + LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + int ml, ml2, ml3, ml0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* start3=NULL; + const BYTE* ref3=NULL; + const BYTE* start0; + const BYTE* ref0; + + + // Ensure blocks follow each other + if (ip != ctx->end) return 0; + ctx->end += inputSize; + + ip++; + + // Main Loop + while (ip < mflimit) + { + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + // saved, in case we would skip too much + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + else ml2 = ml; + + if (ml2 == ml) // No better match + { + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + continue; + } + + if (start0 < ip) + { + if (start2 < ip + ml0) // empirical + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + // Here, start0==ip + if ((start2 - ip) < 3) // First Match too small : removed + { + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + // Currently we have : + // ml2 > ml1, and + // ip1+3 <= ip2 (usually < ip1+ml1) + if ((start2 - ip) < OPTIMAL_ML) + { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) + + if (start2 + ml2 < mflimit) + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + else ml3 = ml2; + + if (ml3 == ml2) // No better match : 2 sequences to encode + { + // ip & ref are known; Now for ml + if (start2 < ip+ml) ml = (int)(start2 - ip); + // Now, encode 2 sequences + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start2; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; + continue; + } + + if (start3 < ip+ml+3) // Not enough space for match 2 : remove it + { + if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 + { + if (start2 < ip+ml) + { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) + { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + // OK, now we have 3 ascending matches; let's write at least the first one + // ip & ref are known; Now for ml + if (start2 < ip+ml) + { + if ((start2 - ip) < (int)ML_MASK) + { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + else + { + ml = (int)(start2 - ip); + } + } + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + + } + + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<<ML_BITS); + memcpy(op, anchor, iend - anchor); + op += iend-anchor; + } + + // End + return (int) (((char*)op)-dest); +} + + +int LZ4_compressHC(const char* source, char* dest, int inputSize) +{ + void* ctx = LZ4_createHC(source); + int result; + if (ctx==NULL) return 0; + + result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit); + + LZ4_freeHC(ctx); + return result; +} + +int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + void* ctx = LZ4_createHC(source); + int result; + if (ctx==NULL) return 0; + + result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput); + + LZ4_freeHC(ctx); + return result; +} + + +//***************************** +// Using an external allocation +//***************************** + +int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); } + + +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize) +{ + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits) + LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); + return LZ4HC_compress_generic (state, source, dest, inputSize, 0, noLimit); +} + + +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits) + LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); + return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, limitedOutput); +} + + +//**************************** +// Stream functions +//**************************** + +int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit); +} + +int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput); +} + @@ -1,111 +1,157 @@ -/*
- LZ4 HC - High Compression Mode of LZ4
- Header File
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-#pragma once
-
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-int LZ4_compressHC (const char* source, char* dest, int inputSize);
-/*
-LZ4_compressHC :
- return : the number of bytes in compressed buffer dest
- or 0 if compression fails.
- note : destination buffer must be already allocated.
- To avoid any problem, size it to handle worst cases situations (input data not compressible)
- Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
-*/
-
-int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-/*
-LZ4_compress_limitedOutput() :
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
- If it cannot achieve it, compression will stop, and result of the function will be zero.
- This function never writes outside of provided output buffer.
-
- inputSize : Max supported value is 1 GB
- maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
- return : the number of output bytes written in buffer 'dest'
- or 0 if compression fails.
-*/
-
-
-/* Note :
-Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
-*/
-
-
-/* Advanced Functions */
-
-void* LZ4_createHC (const char* inputBuffer);
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-int LZ4_freeHC (void* LZ4HC_Data);
-
-/*
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
-In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
-
-void* LZ4_createHC (const char* inputBuffer);
-The result of the function is the (void*) pointer on the LZ4HC Data Structure.
-This pointer will be needed in all other functions.
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
-To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
-Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
-but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
-If next block does not begin immediately after the previous one, the compression will fail (return 0).
-
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
-==> The memory position where the next input data block must start is provided as the result of the function.
-
-Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
-
-When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
-*/
-
-
-#if defined (__cplusplus)
-}
-#endif
+/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + + +#if defined (__cplusplus) +extern "C" { +#endif + + +int LZ4_compressHC (const char* source, char* dest, int inputSize); +/* +LZ4_compressHC : + return : the number of bytes in compressed buffer dest + or 0 if compression fails. + note : destination buffer must be already allocated. + To avoid any problem, size it to handle worst cases situations (input data not compressible) + Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") +*/ + +int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +/* +LZ4_compress_limitedOutput() : + Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. + If it cannot achieve it, compression will stop, and result of the function will be zero. + This function never writes outside of provided output buffer. + + inputSize : Max supported value is 1 GB + maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) + return : the number of output bytes written in buffer 'dest' + or 0 if compression fails. +*/ + + +/* Note : +Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) +*/ + + +//***************************** +// Using an external allocation +//***************************** +int LZ4_sizeofStateHC(); +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* +These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. +To know how much memory must be allocated for the compression tables, use : +int LZ4_sizeofStateHC(); + +Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). + +The allocated memory can be provided to the compressions functions using 'void* state' parameter. +LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. +They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). +*/ + + +//**************************** +// Streaming Functions +//**************************** + +void* LZ4_createHC (const char* inputBuffer); +int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); +char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +int LZ4_freeHC (void* LZ4HC_Data); + +/* +These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. +In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : + +void* LZ4_createHC (const char* inputBuffer); +The result of the function is the (void*) pointer on the LZ4HC Data Structure. +This pointer will be needed in all other functions. +If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. +The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. +To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). +Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), +but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. +If next block does not begin immediately after the previous one, the compression will fail (return 0). + +When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : +char* LZ4_slideInputBufferHC(void* LZ4HC_Data); +must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. +Note that, for this function to work properly, minimum size of an input buffer must be 192KB. +==> The memory position where the next input data block must start is provided as the result of the function. + +Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. + +When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. +*/ + +int LZ4_sizeofStreamStateHC(); +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); + +/* +These functions achieve the same result as : +void* LZ4_createHC (const char* inputBuffer); + +They are provided here to allow the user program to allocate memory using its own routines. + +To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); +Note also that space must be aligned for pointers (32 or 64 bits). + +Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); +void* state is a pointer to the space allocated. +It must be aligned for pointers (32 or 64 bits), and be large enough. +The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). +return value of LZ4_resetStreamStateHC() must be 0 is OK. +Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). +*/ + + +#if defined (__cplusplus) +} +#endif @@ -1,475 +1,475 @@ -/*
-xxHash - Fast Hash algorithm
-Copyright (C) 2012-2013, Yann Collet.
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash source repository : http://code.google.com/p/xxhash/
-*/
-
-
-//**************************************
-// Tuning parameters
-//**************************************
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
-// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
-#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-# define XXH_USE_UNALIGNED_ACCESS 1
-#endif
-
-// XXH_ACCEPT_NULL_INPUT_POINTER :
-// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
-// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
-// This option has a very small performance cost (only measurable on small inputs).
-// By default, this option is disabled. To enable it, uncomment below define :
-//#define XXH_ACCEPT_NULL_INPUT_POINTER 1
-
-// XXH_FORCE_NATIVE_FORMAT :
-// By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
-// Results are therefore identical for little-endian and big-endian CPU.
-// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
-// Should endian-independance be of no importance for your application, you may set the #define below to 1.
-// It will improve speed for Big-endian CPU.
-// This option has no impact on Little_Endian CPU.
-#define XXH_FORCE_NATIVE_FORMAT 0
-
-
-//**************************************
-// Compiler Specific Options
-//**************************************
-// Disable some Visual warning messages
-#ifdef _MSC_VER // Visual Studio
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-#endif
-
-#ifdef _MSC_VER // Visual Studio
-# define forceinline static __forceinline
-#else
-# ifdef __GNUC__
-# define forceinline static inline __attribute__((always_inline))
-# else
-# define forceinline static inline
-# endif
-#endif
-
-
-//**************************************
-// Includes & Memory related functions
-//**************************************
-#include "xxhash.h"
-// Modify the local functions below should you wish to use some other memory related routines
-// for malloc(), free()
-#include <stdlib.h>
-forceinline void* XXH_malloc(size_t s) { return malloc(s); }
-forceinline void XXH_free (void* p) { free(p); }
-// for memcpy()
-#include <string.h>
-forceinline void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
-
-
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)
-# define _PACKED __attribute__ ((packed))
-#else
-# define _PACKED
-#endif
-
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# ifdef __IBMC__
-# pragma pack(1)
-# else
-# pragma pack(push, 1)
-# endif
-#endif
-
-typedef struct _U32_S { U32 v; } _PACKED U32_S;
-
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# pragma pack(pop)
-#endif
-
-#define A32(x) (((U32_S *)(x))->v)
-
-
-//***************************************
-// Compiler-specific Functions and Macros
-//***************************************
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-
-// Note : although _rotl exists for minGW (GCC under windows), performance seems poor
-#if defined(_MSC_VER)
-# define XXH_rotl32(x,r) _rotl(x,r)
-#else
-# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-#endif
-
-#if defined(_MSC_VER) // Visual Studio
-# define XXH_swap32 _byteswap_ulong
-#elif GCC_VERSION >= 403
-# define XXH_swap32 __builtin_bswap32
-#else
-static inline U32 XXH_swap32 (U32 x) {
- return ((x << 24) & 0xff000000 ) |
- ((x << 8) & 0x00ff0000 ) |
- ((x >> 8) & 0x0000ff00 ) |
- ((x >> 24) & 0x000000ff );}
-#endif
-
-
-//**************************************
-// Constants
-//**************************************
-#define PRIME32_1 2654435761U
-#define PRIME32_2 2246822519U
-#define PRIME32_3 3266489917U
-#define PRIME32_4 668265263U
-#define PRIME32_5 374761393U
-
-
-//**************************************
-// Architecture Macros
-//**************************************
-typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
-#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch
- static const int one = 1;
-# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))
-#endif
-
-
-//**************************************
-// Macros
-//**************************************
-#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations
-
-
-//****************************
-// Memory reads
-//****************************
-typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
-
-forceinline U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align)
-{
- if (align==XXH_unaligned)
- return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr));
- else
- return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr);
-}
-
-forceinline U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); }
-
-
-//****************************
-// Simple Hash Functions
-//****************************
-forceinline U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
- U32 h32;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; }
-#endif
-
- if (len>=16)
- {
- const BYTE* const limit = bEnd - 16;
- U32 v1 = seed + PRIME32_1 + PRIME32_2;
- U32 v2 = seed + PRIME32_2;
- U32 v3 = seed + 0;
- U32 v4 = seed - PRIME32_1;
-
- do
- {
- v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
- v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
- v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
- v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
- } while (p<=limit);
-
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
- }
- else
- {
- h32 = seed + PRIME32_5;
- }
-
- h32 += (U32) len;
-
- while (p<=bEnd-4)
- {
- h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
- p+=4;
- }
-
- while (p<bEnd)
- {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
-
- return h32;
-}
-
-
-U32 XXH32(const void* input, int len, U32 seed)
-{
-#if 0
- // Simple version, good for code maintenance, but unfortunately slow for small inputs
- void* state = XXH32_init(seed);
- XXH32_update(state, input, len);
- return XXH32_digest(state);
-#else
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-# if !defined(XXH_USE_UNALIGNED_ACCESS)
- if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage
- {
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- }
-# endif
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-
-//****************************
-// Advanced Hash Functions
-//****************************
-
-struct XXH_state32_t
-{
- U64 total_len;
- U32 seed;
- U32 v1;
- U32 v2;
- U32 v3;
- U32 v4;
- int memsize;
- char memory[16];
-};
-
-
-int XXH32_sizeofState()
-{
- XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough
- return sizeof(struct XXH_state32_t);
-}
-
-
-XXH_errorcode XXH32_resetState(void* state_in, U32 seed)
-{
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
- state->seed = seed;
- state->v1 = seed + PRIME32_1 + PRIME32_2;
- state->v2 = seed + PRIME32_2;
- state->v3 = seed + 0;
- state->v4 = seed - PRIME32_1;
- state->total_len = 0;
- state->memsize = 0;
- return XXH_OK;
-}
-
-
-void* XXH32_init (U32 seed)
-{
- void* state = XXH_malloc (sizeof(struct XXH_state32_t));
- XXH32_resetState(state, seed);
- return state;
-}
-
-
-forceinline XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian)
-{
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
- const BYTE* p = (const BYTE*)input;
- const BYTE* const bEnd = p + len;
-
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (input==NULL) return XXH_ERROR;
-#endif
-
- state->total_len += len;
-
- if (state->memsize + len < 16) // fill in tmp buffer
- {
- XXH_memcpy(state->memory + state->memsize, input, len);
- state->memsize += len;
- return XXH_OK;
- }
-
- if (state->memsize) // some data left from previous update
- {
- XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize);
- {
- const U32* p32 = (const U32*)state->memory;
- state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++;
- state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++;
- state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++;
- state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++;
- }
- p += 16-state->memsize;
- state->memsize = 0;
- }
-
- if (p <= bEnd-16)
- {
- const BYTE* const limit = bEnd - 16;
- U32 v1 = state->v1;
- U32 v2 = state->v2;
- U32 v3 = state->v3;
- U32 v4 = state->v4;
-
- do
- {
- v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
- v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
- v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
- v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
- } while (p<=limit);
-
- state->v1 = v1;
- state->v2 = v2;
- state->v3 = v3;
- state->v4 = v4;
- }
-
- if (p < bEnd)
- {
- XXH_memcpy(state->memory, p, bEnd-p);
- state->memsize = (int)(bEnd-p);
- }
-
- return XXH_OK;
-}
-
-XXH_errorcode XXH32_update (void* state_in, const void* input, int len)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
- else
- return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
-}
-
-
-
-forceinline U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian)
-{
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
- const BYTE * p = (const BYTE*)state->memory;
- BYTE* bEnd = (BYTE*)state->memory + state->memsize;
- U32 h32;
-
- if (state->total_len >= 16)
- {
- h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
- }
- else
- {
- h32 = state->seed + PRIME32_5;
- }
-
- h32 += (U32) state->total_len;
-
- while (p<=bEnd-4)
- {
- h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3;
- h32 = XXH_rotl32(h32, 17) * PRIME32_4;
- p+=4;
- }
-
- while (p<bEnd)
- {
- h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;
- p++;
- }
-
- h32 ^= h32 >> 15;
- h32 *= PRIME32_2;
- h32 ^= h32 >> 13;
- h32 *= PRIME32_3;
- h32 ^= h32 >> 16;
-
- return h32;
-}
-
-
-U32 XXH32_intermediateDigest (void* state_in)
-{
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian);
- else
- return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian);
-}
-
-
-U32 XXH32_digest (void* state_in)
-{
- U32 h32 = XXH32_intermediateDigest(state_in);
-
- XXH_free(state_in);
-
- return h32;
-}
+/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory related routines +// for malloc(), free() +#include <stdlib.h> +FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } +FORCE_INLINE void XXH_free (void* p) { free(p); } +// for memcpy() +#include <string.h> +FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S { U32 v; } _PACKED U32_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static inline U32 XXH_swap32 (U32 x) { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff );} +#endif + + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch + static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } + + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U32 h32; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p<bEnd) + { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32(const void* input, int len, U32 seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + void* state = XXH32_init(seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +//**************************** +// Advanced Hash Functions +//**************************** + +struct XXH_state32_t +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + int memsize; + char memory[16]; +}; + + +int XXH32_sizeofState() +{ + XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough + return sizeof(struct XXH_state32_t); +} + + +XXH_errorcode XXH32_resetState(void* state_in, U32 seed) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +void* XXH32_init (U32 seed) +{ + void* state = XXH_malloc (sizeof(struct XXH_state32_t)); + XXH32_resetState(state, seed); + return state; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + { + const U32* p32 = (const U32*)state->memory; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (void* state_in, const void* input, int len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p<bEnd) + { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_intermediateDigest (void* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); + else + return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); +} + + +U32 XXH32_digest (void* state_in) +{ + U32 h32 = XXH32_intermediateDigest(state_in); + + XXH_free(state_in); + + return h32; +} @@ -1,164 +1,164 @@ -/*
- xxHash - Fast Hash algorithm
- Header File
- Copyright (C) 2012-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - xxHash source repository : http://code.google.com/p/xxhash/
-*/
-
-/* Notice extracted from xxHash homepage :
-
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
-It also successfully passes all tests from the SMHasher suite.
-
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
-
-Name Speed Q.Score Author
-xxHash 5.4 GB/s 10
-CrapWow 3.2 GB/s 2 Andrew
-MumurHash 3a 2.7 GB/s 10 Austin Appleby
-SpookyHash 2.0 GB/s 10 Bob Jenkins
-SBox 1.4 GB/s 9 Bret Mulvey
-Lookup3 1.2 GB/s 9 Bob Jenkins
-SuperFastHash 1.2 GB/s 1 Paul Hsieh
-CityHash64 1.05 GB/s 10 Pike & Alakuijala
-FNV 0.55 GB/s 5 Fowler, Noll, Vo
-CRC32 0.43 GB/s 9
-MD5-32 0.33 GB/s 10 Ronald L. Rivest
-SHA1-32 0.28 GB/s 10
-
-Q.Score is a measure of quality of the hash function.
-It depends on successfully passing SMHasher test set.
-10 is a perfect score.
-*/
-
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-//****************************
-// Type
-//****************************
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
-
-
-
-//****************************
-// Simple Hash Functions
-//****************************
-
-unsigned int XXH32 (const void* input, int len, unsigned int seed);
-
-/*
-XXH32() :
- Calculate the 32-bits hash of sequence of length "len" stored at memory address "input".
- The memory between input & input+len must be valid (allocated and read-accessible).
- "seed" can be used to alter the result predictably.
- This function successfully passes all SMHasher tests.
- Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
- Note that "len" is type "int", which means it is limited to 2^31-1.
- If your data is larger, use the advanced functions below.
-*/
-
-
-
-//****************************
-// Advanced Hash Functions
-//****************************
-
-void* XXH32_init (unsigned int seed);
-XXH_errorcode XXH32_update (void* state, const void* input, int len);
-unsigned int XXH32_digest (void* state);
-
-/*
-These functions calculate the xxhash of an input provided in several small packets,
-as opposed to an input provided as a single block.
-
-It must be started with :
-void* XXH32_init()
-The function returns a pointer which holds the state of calculation.
-
-This pointer must be provided as "void* state" parameter for XXH32_update().
-XXH32_update() can be called as many times as necessary.
-The user must provide a valid (allocated) input.
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
-Note that "len" is type "int", which means it is limited to 2^31-1.
-If your data is larger, it is recommended to chunk your data into blocks
-of size for example 2^30 (1GB) to avoid any "int" overflow issue.
-
-Finally, you can end the calculation anytime, by using XXH32_digest().
-This function returns the final 32-bits hash.
-You must provide the same "void* state" parameter created by XXH32_init().
-Memory will be freed by XXH32_digest().
-*/
-
-
-int XXH32_sizeofState();
-XXH_errorcode XXH32_resetState(void* state, unsigned int seed);
-
-#define XXH32_SIZEOFSTATE 48
-typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t;
-/*
-These functions allow user application to make its own allocation for state.
-
-XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state.
-Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer.
-This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state.
-
-For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()),
-use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields.
-*/
-
-
-unsigned int XXH32_intermediateDigest (void* state);
-/*
-This function does the same as XXH32_digest(), generating a 32-bit hash,
-but preserve memory context.
-This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update().
-To free memory context, use XXH32_digest(), or free().
-*/
-
-
-
-//****************************
-// Deprecated function names
-//****************************
-// The following translations are provided to ease code transition
-// You are encouraged to no longer this function names
-#define XXH32_feed XXH32_update
-#define XXH32_result XXH32_digest
-#define XXH32_getIntermediateResult XXH32_intermediateDigest
-
-
-
-#if defined (__cplusplus)
-}
-#endif
+/* + xxHash - Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +//**************************** +// Type +//**************************** +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +//**************************** +// Simple Hash Functions +//**************************** + +unsigned int XXH32 (const void* input, int len, unsigned int seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". + The memory between input & input+len must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s + Note that "len" is type "int", which means it is limited to 2^31-1. + If your data is larger, use the advanced functions below. +*/ + + + +//**************************** +// Advanced Hash Functions +//**************************** + +void* XXH32_init (unsigned int seed); +XXH_errorcode XXH32_update (void* state, const void* input, int len); +unsigned int XXH32_digest (void* state); + +/* +These functions calculate the xxhash of an input provided in several small packets, +as opposed to an input provided as a single block. + +It must be started with : +void* XXH32_init() +The function returns a pointer which holds the state of calculation. + +This pointer must be provided as "void* state" parameter for XXH32_update(). +XXH32_update() can be called as many times as necessary. +The user must provide a valid (allocated) input. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. +Note that "len" is type "int", which means it is limited to 2^31-1. +If your data is larger, it is recommended to chunk your data into blocks +of size for example 2^30 (1GB) to avoid any "int" overflow issue. + +Finally, you can end the calculation anytime, by using XXH32_digest(). +This function returns the final 32-bits hash. +You must provide the same "void* state" parameter created by XXH32_init(). +Memory will be freed by XXH32_digest(). +*/ + + +int XXH32_sizeofState(); +XXH_errorcode XXH32_resetState(void* state, unsigned int seed); + +#define XXH32_SIZEOFSTATE 48 +typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; +/* +These functions allow user application to make its own allocation for state. + +XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. +Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. +This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. + +For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), +use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. +*/ + + +unsigned int XXH32_intermediateDigest (void* state); +/* +This function does the same as XXH32_digest(), generating a 32-bit hash, +but preserve memory context. +This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). +To free memory context, use XXH32_digest(), or free(). +*/ + + + +//**************************** +// Deprecated function names +//**************************** +// The following translations are provided to ease code transition +// You are encouraged to no longer this function names +#define XXH32_feed XXH32_update +#define XXH32_result XXH32_digest +#define XXH32_getIntermediateResult XXH32_intermediateDigest + + + +#if defined (__cplusplus) +} +#endif |