summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoryann.collet.73@gmail.com <yann.collet.73@gmail.com@650e7d94-2a16-8b24-b05c-7c0b3f6821cd>2014-02-04 14:11:10 (GMT)
committeryann.collet.73@gmail.com <yann.collet.73@gmail.com@650e7d94-2a16-8b24-b05c-7c0b3f6821cd>2014-02-04 14:11:10 (GMT)
commit69dc85b8abe78246bea91a5ba1205e4c07b96a97 (patch)
tree0ed0250f8284861b90c2781aaa76172ebf97b6fd
parent96a60a52eb3e7d7a4785ad1dfc4e0abe3d0e10a1 (diff)
downloadlz4-69dc85b8abe78246bea91a5ba1205e4c07b96a97.zip
lz4-69dc85b8abe78246bea91a5ba1205e4c07b96a97.tar.gz
lz4-69dc85b8abe78246bea91a5ba1205e4c07b96a97.tar.bz2
Large decompression speed improvement for GCC 32-bits. Thanks to Valery Croizier !
LZ4HC : Compression Level is now a programmable parameter (CLI from 4 to 9) Separated IO routines from command line (lz4io.c) Version number into lz4.h (suggested by Francesc Alted) git-svn-id: https://lz4.googlecode.com/svn/trunk@113 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
-rw-r--r--Makefile3
-rw-r--r--NEWS6
-rw-r--r--cmake_unofficial/CMakeLists.txt2
-rw-r--r--lz4.c374
-rw-r--r--lz4.h500
-rw-r--r--lz4hc.c306
-rw-r--r--lz4hc.h329
-rw-r--r--programs/Makefile8
-rw-r--r--programs/bench.c12
-rw-r--r--programs/fuzzer.c5
-rw-r--r--programs/lz4cli.c773
-rw-r--r--programs/lz4io.c948
-rw-r--r--programs/lz4io.h77
13 files changed, 1855 insertions, 1488 deletions
diff --git a/Makefile b/Makefile
index 2fe42f9..a24e503 100644
--- a/Makefile
+++ b/Makefile
@@ -30,7 +30,7 @@
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
-RELEASE=r112
+RELEASE=r113
DESTDIR=
PREFIX=/usr
CC=gcc
@@ -53,6 +53,7 @@ TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \
lz4_format_description.txt Makefile NEWS LICENSE \
cmake_unofficial/CMakeLists.txt \
$(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \
+ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
$(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \
$(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING
diff --git a/NEWS b/NEWS
index a83aebe..23d059e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+r113:
+Large decompression speed improvement for GCC 32-bits. Thanks to Valery Croizier !
+LZ4HC : Compression Level is now a programmable parameter (CLI from 4 to 9)
+Separated IO routines from command line (lz4io.c)
+Version number into lz4.h (suggested by Francesc Alted)
+
r112:
quickfix
diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt
index 356f18a..480654e 100644
--- a/cmake_unofficial/CMakeLists.txt
+++ b/cmake_unofficial/CMakeLists.txt
@@ -27,7 +27,7 @@ endif()
set(LZ4_DIR ../)
set(PRG_DIR ../programs/)
set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h)
-set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c)
+set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c)
if(BUILD_TOOLS AND NOT BUILD_LIBS)
set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})
diff --git a/lz4.c b/lz4.c
index f521b0f..ee37895 100644
--- a/lz4.c
+++ b/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -31,37 +31,43 @@
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-//**************************************
-// Tuning parameters
-//**************************************
-// MEMORY_USAGE :
-// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-// Increasing memory usage improves compression ratio
-// Reduced memory usage can improve speed, due to cache effect
-// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+/**************************************
+ Tuning parameters
+**************************************/
+/*
+ * MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
#define MEMORY_USAGE 14
-// HEAPMODE :
-// Select how default compression functions will allocate memory for their hash table,
-// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
+/*
+ * HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
+ */
#define HEAPMODE 0
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
+/**************************************
+ CPU Feature Detection
+**************************************/
+/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
# define LZ4_ARCH64 0
#endif
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
+/*
+ * Little Endian or Big Endian ?
+ * Overwrite the #define below if you know your architecture endianess
+ */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)
@@ -75,48 +81,53 @@
|| defined(_MIPSEB) || defined(__s390__)
# define LZ4_BIG_ENDIAN 1
#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */
#endif
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+/*
+ * Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+ */
#if defined(__ARM_FEATURE_UNALIGNED)
# define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+/* Define this parameter if your target system or compiler does not support hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LZ4_FORCE_SW_BITCOUNT
#endif
-// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
-// This option may provide a small boost to performance for some big endian cpu, although probably modest.
-// You may set this option to 1 if data will remain within closed environment.
-// This option is useless on Little_Endian CPU (such as x86)
-//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1
+/*
+ * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
+ * This option may provide a small boost to performance for some big endian cpu, although probably modest.
+ * You may set this option to 1 if data will remain within closed environment.
+ * This option is useless on Little_Endian CPU (such as x86)
+ */
+
+/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99
+/**************************************
+ Compiler Options
+**************************************/
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
/* "restrict" is a known keyword */
#else
-# define restrict // Disable restrict
+# define restrict /* Disable restrict */
#endif
-#ifdef _MSC_VER // Visual Studio
+#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
+# include <intrin.h> /* For Visual 2005 */
+# if LZ4_ARCH64 /* 64-bits */
+# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */
+# else /* 32-bits */
+# pragma intrinsic(_BitScanForward) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */
# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#else
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
@@ -125,7 +136,7 @@
# endif
#endif
-#ifdef _MSC_VER
+#ifdef _MSC_VER /* Visual Studio */
# define lz4_bswap16(x) _byteswap_ushort(x)
#else
# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
@@ -143,26 +154,26 @@
#define unlikely(expr) expect((expr) != 0, 0)
-//**************************************
-// Memory routines
-//**************************************
-#include <stdlib.h> // malloc, calloc, free
+/**************************************
+ Memory routines
+**************************************/
+#include <stdlib.h> /* malloc, calloc, free */
#define ALLOCATOR(n,s) calloc(n,s)
#define FREEMEM free
-#include <string.h> // memset, memcpy
+#include <string.h> /* memset, memcpy */
#define MEM_INIT memset
-//**************************************
-// Includes
-//**************************************
+/**************************************
+ Includes
+**************************************/
#include "lz4.h"
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+/**************************************
+ Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -210,9 +221,9 @@ typedef struct {size_t v;} _PACKED size_t_S;
#define AARCH(x) (((size_t_S *)(x))->v)
-//**************************************
-// Constants
-//**************************************
+/**************************************
+ Constants
+**************************************/
#define LZ4_HASHLOG (MEMORY_USAGE-2)
#define HASHTABLESIZE (1 << MEMORY_USAGE)
#define HASHNBCELLS4 (1 << LZ4_HASHLOG)
@@ -222,10 +233,14 @@ typedef struct {size_t v;} _PACKED size_t_S;
#define COPYLENGTH 8
#define LASTLITERALS 5
#define MFLIMIT (COPYLENGTH+MINMATCH)
-const int LZ4_minLength = (MFLIMIT+1);
+static const int LZ4_minLength = (MFLIMIT+1);
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))
-#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data
+#define LZ4_64KLIMIT ((64 KB) + (MFLIMIT-1))
+#define SKIPSTRENGTH 6 /* Increasing this value will make the compression run slower on incompressible data */
#define MAXD_LOG 16
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
@@ -235,15 +250,10 @@ const int LZ4_minLength = (MFLIMIT+1);
#define RUN_BITS (8-ML_BITS)
#define RUN_MASK ((1U<<RUN_BITS)-1)
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-
-//**************************************
-// Structures and local types
-//**************************************
+/**************************************
+ Structures and local types
+**************************************/
typedef struct {
U32 hashTable[HASHNBCELLS4];
const BYTE* bufferStart;
@@ -260,40 +270,36 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
-//**************************************
-// Architecture-specific macros
-//**************************************
+/**************************************
+ Architecture-specific macros
+**************************************/
#define STEPSIZE sizeof(size_t)
#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }
#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); }
-#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); }
-
-#if LZ4_ARCH64 // 64-bit
-# define HTYPE U32
-# define INITBASE(base) const BYTE* const base = ip
-#else // 32-bit
-# define HTYPE const BYTE*
-# define INITBASE(base) const int base = 0
-#endif
#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
+#else /* Little Endian */
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
#endif
-//**************************************
-// Macros
-//**************************************
-#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e;
+/**************************************
+ Macros
+**************************************/
+#if LZ4_ARCH64 || !defined(__GNUC__)
+# define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } /* at the end, d>=e; */
+#else
+# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
+#endif
+#define LZ4_SECURECOPY(d,s,e) { if (d<e) LZ4_WILDCOPY(d,s,e); }
-//****************************
-// Private functions
-//****************************
+/****************************
+ Private local functions
+****************************/
#if LZ4_ARCH64
FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
@@ -360,9 +366,9 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
#endif
-//****************************
-// Compression functions
-//****************************
+/****************************
+ Compression functions
+****************************/
FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
{
if (tableType == byU16)
@@ -393,7 +399,7 @@ FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType
{
if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
- { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return
+ { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
}
FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
@@ -429,18 +435,18 @@ FORCE_INLINE int LZ4_compress_generic(
const int skipStrength = SKIPSTRENGTH;
U32 forwardH;
- // Init conditions
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative)
- if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block
- if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit
- if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit)
- if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals)
+ /* Init conditions */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
+ if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; /* must continue from end of previous block */
+ if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; /* do it now, due to potential early exit */
+ if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */
+ if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
- // First Byte
+ /* First Byte */
LZ4_putPosition(ip, ctx, tableType, base);
ip++; forwardH = LZ4_hashPosition(ip, tableType);
- // Main Loop
+ /* Main Loop */
for ( ; ; )
{
int findMatchAttempts = (1U << skipStrength) + 3;
@@ -448,14 +454,14 @@ FORCE_INLINE int LZ4_compress_generic(
const BYTE* ref;
BYTE* token;
- // Find a match
+ /* Find a match */
do {
U32 h = forwardH;
int step = findMatchAttempts++ >> skipStrength;
ip = forwardIp;
forwardIp = ip + step;
- if unlikely(forwardIp > mflimit) { goto _last_literals; }
+ if (unlikely(forwardIp > mflimit)) { goto _last_literals; }
forwardH = LZ4_hashPosition(forwardIp, tableType);
ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
@@ -463,13 +469,13 @@ FORCE_INLINE int LZ4_compress_generic(
} while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));
- // Catch up
- while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; }
+ /* Catch up */
+ while ((ip>anchor) && (ref > lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; }
- // Encode Literal length
+ /* Encode Literal length */
length = (int)(ip - anchor);
token = op++;
- if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit
+ if ((limitedOutput) && (unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend))) return 0; /* Check output limit */
if (length>=(int)RUN_MASK)
{
int len = length-RUN_MASK;
@@ -479,17 +485,17 @@ FORCE_INLINE int LZ4_compress_generic(
}
else *token = (BYTE)(length<<ML_BITS);
- // Copy Literals
+ /* Copy Literals */
{ BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }
_next_match:
- // Encode Offset
+ /* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
- // Start Counting
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified
+ /* Start Counting */
+ ip+=MINMATCH; ref+=MINMATCH; /* MinMatch already verified */
anchor = ip;
- while likely(ip<matchlimit-(STEPSIZE-1))
+ while (likely(ip<matchlimit-(STEPSIZE-1)))
{
size_t diff = AARCH(ref) ^ AARCH(ip);
if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
@@ -501,9 +507,9 @@ _next_match:
if ((ip<matchlimit) && (*ref == *ip)) ip++;
_endCount:
- // Encode MatchLength
+ /* Encode MatchLength */
length = (int)(ip - anchor);
- if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit
+ if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend))) return 0; /* Check output limit */
if (length>=(int)ML_MASK)
{
*token += ML_MASK;
@@ -514,34 +520,34 @@ _endCount:
}
else *token += (BYTE)(length);
- // Test end of chunk
+ /* Test end of chunk */
if (ip > mflimit) { anchor = ip; break; }
- // Fill table
+ /* Fill table */
LZ4_putPosition(ip-2, ctx, tableType, base);
- // Test next position
+ /* Test next position */
ref = LZ4_getPosition(ip, ctx, tableType, base);
LZ4_putPosition(ip, ctx, tableType, base);
if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
- // Prepare next loop
+ /* Prepare next loop */
anchor = ip++;
forwardH = LZ4_hashPosition(ip, tableType);
}
_last_literals:
- // Encode Last Literals
+ /* Encode Last Literals */
{
int lastRun = (int)(iend - anchor);
- if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
+ if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
}
- // End
+ /* End */
return (int) (((char*)op)-dest);
}
@@ -549,9 +555,9 @@ _last_literals:
int LZ4_compress(const char* source, char* dest, int inputSize)
{
#if (HEAPMODE)
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
+ void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */
#else
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
+ U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
#endif
int result;
@@ -569,9 +575,9 @@ int LZ4_compress(const char* source, char* dest, int inputSize)
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
{
#if (HEAPMODE)
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
+ void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */
#else
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
+ U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
#endif
int result;
@@ -587,16 +593,16 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
}
-//*****************************
-// Using an external allocation
-//*****************************
+/*****************************
+ Using external allocation
+*****************************/
int LZ4_sizeofState() { return 1 << MEMORY_USAGE; }
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
{
- if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary
+ if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
@@ -608,7 +614,7 @@ int LZ4_compress_withState (void* state, const char* source, char* dest, int inp
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{
- if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary
+ if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
MEM_INIT(state, 0, LZ4_sizeofState());
if (inputSize < (int)LZ4_64KLIMIT)
@@ -618,9 +624,9 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
}
-//****************************
-// Stream functions
-//****************************
+/****************************
+ Stream functions
+****************************/
int LZ4_sizeofStreamState()
{
@@ -637,7 +643,7 @@ FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
int LZ4_resetStreamState(void* state, const char* inputBuffer)
{
- if ((((size_t)state) & 3) != 0) return 1; // Error : pointer is not aligned on 4-bytes boundary
+ if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer);
return 0;
}
@@ -662,8 +668,8 @@ char* LZ4_slideInputBuffer (void* LZ4_Data)
LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;
size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);
- if ( (lz4ds->base - delta > lz4ds->base) // underflow control
- || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit
+ if ( (lz4ds->base - delta > lz4ds->base) /* underflow control */
+ || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) /* close to 32-bits limit */
{
size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;
int nH;
@@ -700,27 +706,29 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha
}
-//****************************
-// Decompression functions
-//****************************
+/****************************
+ Decompression functions
+****************************/
-// This generic decompression function cover all use cases.
-// It shall be instanciated several times, using different sets of directives
-// Note that it is essential this generic function is really inlined,
-// in order to remove useless branches during compilation optimisation.
+/*
+ * This generic decompression function cover all use cases.
+ * It shall be instanciated several times, using different sets of directives
+ * Note that it is essential this generic function is really inlined,
+ * in order to remove useless branches during compilation optimisation.
+ */
FORCE_INLINE int LZ4_decompress_generic(
const char* source,
char* dest,
- int inputSize, //
- int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer.
+ int inputSize,
+ int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
- int endOnInput, // endOnOutputSize, endOnInputSize
- int prefix64k, // noPrefix, withPrefix
- int partialDecoding, // full, partial
- int targetOutputSize // only used if partialDecoding==partial
+ int endOnInput, /* endOnOutputSize, endOnInputSize */
+ int prefix64k, /* noPrefix, withPrefix */
+ int partialDecoding, /* full, partial */
+ int targetOutputSize /* only used if partialDecoding==partial */
)
{
- // Local Variables
+ /* Local Variables */
const BYTE* restrict ip = (const BYTE*) source;
const BYTE* ref;
const BYTE* const iend = ip + inputSize;
@@ -730,23 +738,24 @@ FORCE_INLINE int LZ4_decompress_generic(
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
- const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64
+ /*const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; / static reduces speed for LZ4_decompress_safe() on GCC64 */
+ const size_t dec32table[] = {4-0, 4-3, 4-2, 4-3, 4-0, 4-0, 4-0, 4-0}; /* static reduces speed for LZ4_decompress_safe() on GCC64 */
static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
- // Special cases
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything
- if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer
- if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1);
+ /* Special cases */
+ if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
+ if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
+ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
- // Main Loop
+ /* Main Loop */
while (1)
{
unsigned token;
size_t length;
- // get runlength
+ /* get runlength */
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK)
{
@@ -758,36 +767,36 @@ FORCE_INLINE int LZ4_decompress_generic(
}
}
- // copy literals
+ /* copy literals */
cpy = op+length;
if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
|| ((!endOnInput) && (cpy>oend-COPYLENGTH)))
{
if (partialDecoding)
{
- if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer
- if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer
+ if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */
+ if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
}
else
{
- if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed
+ if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
}
memcpy(op, ip, length);
ip += length;
op += length;
- break; // Necessarily EOF, due to parsing restrictions
+ break; /* Necessarily EOF, due to parsing restrictions */
}
LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
- // get offset
+ /* get offset */
LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer
+ if ((prefix64k==noPrefix) && (unlikely(ref < (BYTE* const)dest))) goto _output_error; /* Error : offset outside destination buffer */
- // get matchlength
+ /* get matchlength */
if ((length=(token&ML_MASK)) == ML_MASK)
{
- while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token
+ while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) /* Ensure enough bytes remain for LASTLITERALS + token */
{
unsigned s = *ip++;
length += s;
@@ -796,39 +805,42 @@ FORCE_INLINE int LZ4_decompress_generic(
}
}
- // copy repeated sequence
- if unlikely((op-ref)<(int)STEPSIZE)
+ /* copy repeated sequence */
+ if (unlikely((op-ref)<(int)STEPSIZE))
{
const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref];
op[0] = ref[0];
op[1] = ref[1];
op[2] = ref[2];
op[3] = ref[3];
- op += 4, ref += 4; ref -= dec32table[op-ref];
+ /*op += 4, ref += 4; ref -= dec32table[op-ref];
A32(op) = A32(ref);
- op += STEPSIZE-4; ref -= dec64;
+ op += STEPSIZE-4; ref -= dec64;*/
+ ref += dec32table[op-ref];
+ A32(op+4) = A32(ref);
+ op += STEPSIZE; ref -= dec64;
} else { LZ4_COPYSTEP(op,ref); }
cpy = op + length - (STEPSIZE-4);
- if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4))
+ if (unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4)))
{
- if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
+ if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */
LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
op=cpy;
continue;
}
LZ4_WILDCOPY(op, ref, cpy);
- op=cpy; // correction
+ op=cpy; /* correction */
}
- // end of decoding
+ /* end of decoding */
if (endOnInput)
- return (int) (((char*)op)-dest); // Nb of output bytes decoded
+ return (int) (((char*)op)-dest); /* Nb of output bytes decoded */
else
- return (int) (((char*)ip)-source); // Nb of input bytes read
+ return (int) (((char*)ip)-source); /* Nb of input bytes read */
- // Overflow error detected
+ /* Overflow error detected */
_output_error:
return (int) (-(((char*)ip)-source))-1;
}
@@ -856,7 +868,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int output
int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
{
-#ifdef _MSC_VER // This version is faster with Visual
+#ifdef _MSC_VER /* This version is faster with Visual */
return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0);
#else
return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
diff --git a/lz4.h b/lz4.h
index af05dbc..0bfe44a 100644
--- a/lz4.h
+++ b/lz4.h
@@ -1,249 +1,251 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio
-# define inline __inline // Visual C is not C99, but supports some kind of inline
-#endif
-
-
-//****************************
-// Simple Functions
-//****************************
-
-int LZ4_compress (const char* source, char* dest, int inputSize);
-int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-LZ4_compress() :
- Compresses 'inputSize' bytes from 'source' into 'dest'.
- Destination buffer must be already allocated,
- and must be sized to handle worst cases situations (input data not compressible)
- Worst case size evaluation is provided by function LZ4_compressBound()
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
- return : the number of bytes written in buffer dest
- or 0 if the compression fails
-
-LZ4_decompress_safe() :
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
-*/
-
-
-//****************************
-// Advanced Functions
-//****************************
-#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes
-#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
-static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-
-/*
-LZ4_compressBound() :
- Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
- primarily useful for memory allocation of output buffer.
- inline function is recommended for the general case,
- macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
-
- isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
- return : maximum output size in a "worst case" scenario
- or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
-*/
-
-
-int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-LZ4_compress_limitedOutput() :
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
- If it cannot achieve it, compression will stop, and result of the function will be zero.
- This function never writes outside of provided output buffer.
-
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes written in buffer 'dest'
- or 0 if the compression fails
-*/
-
-
-int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
-
-/*
-LZ4_decompress_fast() :
- outputSize : is the original (uncompressed) size
- return : the number of bytes read from the source buffer (in other words, the compressed size)
- If the source stream is malformed, the function will stop decoding and return a negative result.
- note : This function is a bit faster than LZ4_decompress_safe()
- This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
- Use this function preferably into a trusted environment (data to decode comes from a trusted source).
- Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
-*/
-
-int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
-
-/*
-LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'inputSize' at position 'source'
- into output buffer 'dest' of size 'maxOutputSize'.
- The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
- reducing decompression time.
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
- Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
- Always control how many bytes were decoded.
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
-*/
-
-
-//*****************************
-// Using an external allocation
-//*****************************
-int LZ4_sizeofState();
-int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
-To know how much memory must be allocated for the compression tables, use :
-int LZ4_sizeofState();
-
-Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
-
-The allocated memory can be provided to the compressions functions using 'void* state' parameter.
-LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
-They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
-*/
-
-
-//****************************
-// Streaming Functions
-//****************************
-
-void* LZ4_create (const char* inputBuffer);
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBuffer (void* LZ4_Data);
-int LZ4_free (void* LZ4_Data);
-
-/*
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
-In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
-
-void* LZ4_create (const char* inputBuffer);
-The result of the function is the (void*) pointer on the LZ4 Data Structure.
-This pointer will be needed in all other functions.
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
-To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
-Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
-but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
-If next block does not begin immediately after the previous one, the compression will fail (return 0).
-
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
-char* LZ4_slideInputBuffer(void* LZ4_Data);
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
-==> The memory position where the next input data block must start is provided as the result of the function.
-
-Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
-
-When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
-*/
-
-int LZ4_sizeofStreamState();
-int LZ4_resetStreamState(void* state, const char* inputBuffer);
-
-/*
-These functions achieve the same result as :
-void* LZ4_create (const char* inputBuffer);
-
-They are provided here to allow the user program to allocate memory using its own routines.
-
-To know how much space must be allocated, use LZ4_sizeofStreamState();
-Note also that space must be 4-bytes aligned.
-
-Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
-void* state is a pointer to the space allocated.
-It must be aligned on 4-bytes boundaries, and be large enough.
-The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
-return value of LZ4_resetStreamState() must be 0 is OK.
-Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
-*/
-
-
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
-
-/*
-*_withPrefix64k() :
- These decoding functions work the same as their "normal name" versions,
- but can use up to 64KB of data in front of 'char* dest'.
- These functions are necessary to decode inter-dependant blocks.
-*/
-
-
-//****************************
-// Obsolete Functions
-//****************************
-
-static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
-
-/*
-These functions are deprecated and should no longer be used.
-They are provided here for compatibility with existing user programs.
-*/
-
-
-
-#if defined (__cplusplus)
-}
-#endif
+/*
+ LZ4 - Fast LZ compression algorithm
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/**************************************
+ Version
+**************************************/
+#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */
+#define LZ4_VERSION_MINOR 1 /* for minor interface/format changes */
+#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
+
+
+/**************************************
+ Compiler Options
+**************************************/
+#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus)) /* Visual Studio */
+# define inline __inline /* Visual C is not C99, but supports some kind of inline */
+#endif
+
+
+/**************************************
+ Simple Functions
+**************************************/
+
+int LZ4_compress (const char* source, char* dest, int inputSize);
+int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress() :
+ Compresses 'inputSize' bytes from 'source' into 'dest'.
+ Destination buffer must be already allocated,
+ and must be sized to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound()
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ return : the number of bytes written in buffer dest
+ or 0 if the compression fails
+
+LZ4_decompress_safe() :
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
+*/
+
+
+/**************************************
+ Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
+
+/*
+LZ4_compressBound() :
+ Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
+ primarily useful for memory allocation of output buffer.
+ inline function is recommended for the general case,
+ macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
+
+ isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
+ return : maximum output size in a "worst case" scenario
+ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+
+
+int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes written in buffer 'dest'
+ or 0 if the compression fails
+*/
+
+
+int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
+
+/*
+LZ4_decompress_fast() :
+ outputSize : is the original (uncompressed) size
+ return : the number of bytes read from the source buffer (in other words, the compressed size)
+ If the source stream is malformed, the function will stop decoding and return a negative result.
+ note : This function is a bit faster than LZ4_decompress_safe()
+ This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
+ Use this function preferably into a trusted environment (data to decode comes from a trusted source).
+ Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
+*/
+
+int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
+
+/*
+LZ4_decompress_safe_partial() :
+ This function decompress a compressed block of size 'inputSize' at position 'source'
+ into output buffer 'dest' of size 'maxOutputSize'.
+ The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+ reducing decompression time.
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+ Always control how many bytes were decoded.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+
+
+int LZ4_sizeofState();
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofState();
+
+Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+ Streaming Functions
+**************************************/
+void* LZ4_create (const char* inputBuffer);
+int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBuffer (void* LZ4_Data);
+int LZ4_free (void* LZ4_Data);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
+
+void* LZ4_create (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4 Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
+Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
+but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBuffer(void* LZ4_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
+*/
+
+int LZ4_sizeofStreamState();
+int LZ4_resetStreamState(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_create (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamState();
+Note also that space must be 4-bytes aligned.
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned on 4-bytes boundaries, and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamState() must be 0 is OK.
+Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
+*/
+
+
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
+
+/*
+*_withPrefix64k() :
+ These decoding functions work the same as their "normal name" versions,
+ but can use up to 64KB of data in front of 'char* dest'.
+ These functions are necessary to decode inter-dependant blocks.
+*/
+
+
+/**************************************
+ Obsolete Functions
+**************************************/
+/*
+These functions are deprecated and should no longer be used.
+They are provided here for compatibility with existing user programs.
+*/
+static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
+static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/lz4hc.c b/lz4hc.c
index f28283f..e84de2b 100644
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -31,31 +31,41 @@
- LZ4 source repository : http://code.google.com/p/lz4/
*/
-//**************************************
-// Memory routines
-//**************************************
-#include <stdlib.h> // calloc, free
+
+
+/**************************************
+ Tuning Parameter
+**************************************/
+#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8
+
+
+/**************************************
+ Memory routines
+**************************************/
+#include <stdlib.h> /* calloc, free */
#define ALLOCATOR(s) calloc(1,s)
#define FREEMEM free
-#include <string.h> // memset, memcpy
+#include <string.h> /* memset, memcpy */
#define MEM_INIT memset
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
+/**************************************
+ CPU Feature Detection
+**************************************/
+/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
|| defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
|| defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
# define LZ4_ARCH64 0
#endif
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
+/*
+ * Little Endian or Big Endian ?
+ * Overwrite the #define below if you know your architecture endianess
+ */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)
@@ -69,43 +79,45 @@
|| defined(_MIPSEB) || defined(__s390__)
# define LZ4_BIG_ENDIAN 1
#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */
#endif
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+/*
+ * Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+ */
#if defined(__ARM_FEATURE_UNALIGNED)
# define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+/* Define this parameter if your target system or compiler does not support hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LZ4_FORCE_SW_BITCOUNT
#endif
-//**************************************
-// Compiler Options
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
- /* "restrict" is a known keyword */
+/**************************************
+ Compiler Options
+**************************************/
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
+/* "restrict" is a known keyword */
#else
-# define restrict // Disable restrict
+# define restrict /* Disable restrict */
#endif
-#ifdef _MSC_VER // Visual Studio
+#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
+# include <intrin.h> /* For Visual 2005 */
+# if LZ4_ARCH64 /* 64-bits */
+# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */
+# else /* 32-bits */
+# pragma intrinsic(_BitScanForward) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */
# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */
#else
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
@@ -114,24 +126,24 @@
# endif
#endif
-#ifdef _MSC_VER // Visual Studio
+#ifdef _MSC_VER /* Visual Studio */
# define lz4_bswap16(x) _byteswap_ushort(x)
#else
# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
#endif
-//**************************************
-// Includes
-//**************************************
+/**************************************
+ Includes
+**************************************/
#include "lz4hc.h"
#include "lz4.h"
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+/**************************************
+ Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -173,9 +185,9 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define A16(x) (((U16_S *)(x))->v)
-//**************************************
-// Constants
-//**************************************
+/**************************************
+ Constants
+**************************************/
#define MINMATCH 4
#define DICTIONARY_LOGSIZE 16
@@ -187,8 +199,6 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
-#define MAX_NB_ATTEMPTS 256
-
#define ML_BITS 4
#define ML_MASK (size_t)((1U<<ML_BITS)-1)
#define RUN_BITS (8-ML_BITS)
@@ -205,25 +215,21 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define GB *(1U<<30)
-//**************************************
-// Architecture-specific macros
-//**************************************
-#if LZ4_ARCH64 // 64-bit
+/**************************************
+ Architecture-specific macros
+**************************************/
+#if LZ4_ARCH64 /* 64-bit */
# define STEPSIZE 8
# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
-# define UARCH U64
# define AARCH A64
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
-#else // 32-bit
+#else /* 32-bit */
# define STEPSIZE 4
# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
-# define UARCH U32
# define AARCH A32
-//# define HTYPE const BYTE*
-//# define INITBASE(b,s) const int b = 0
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
#endif
@@ -231,15 +237,15 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#if defined(LZ4_BIG_ENDIAN)
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
+#else /* Little Endian */
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
#endif
-//************************************************************
-// Local Types
-//************************************************************
+/**************************************
+ Local Types
+**************************************/
typedef struct
{
const BYTE* inputBuffer;
@@ -251,9 +257,9 @@ typedef struct
} LZ4HC_Data_Structure;
-//**************************************
-// Macros
-//**************************************
+/**************************************
+ Macros
+**************************************/
#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
@@ -263,9 +269,9 @@ typedef struct
#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
-//**************************************
-// Private functions
-//**************************************
+/**************************************
+ Private functions
+**************************************/
#if LZ4_ARCH64
FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
@@ -349,7 +355,7 @@ FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
{
- if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; // Error : pointer is not aligned for pointer (32 or 64 bits)
+ if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
return 0;
}
@@ -370,7 +376,7 @@ int LZ4_freeHC (void* LZ4HC_Data)
}
-// Update chains up to ip (excluded)
+/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
{
U16* chainTable = hc4->chainTable;
@@ -393,12 +399,12 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;
- distance = (distance >> 16) << 16; // Must be a multiple of 64 KB
+ distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
LZ4HC_Insert(hc4, hc4->end - MINMATCH);
memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
hc4->nextToUpdate -= distance;
hc4->base -= distance;
- if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow
+ if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
{
int i;
hc4->base += 1 GB;
@@ -415,7 +421,7 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY
while (p1t<matchlimit-(STEPSIZE-1))
{
- UARCH diff = AARCH(p2) ^ AARCH(p1t);
+ size_t diff = AARCH(p2) ^ AARCH(p1t);
if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }
p1t += LZ4_NbCommonBytes(diff);
return (p1t - p1);
@@ -427,26 +433,26 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY
}
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
const BYTE* ref;
INITBASE(base,hc4->base);
- int nbAttempts=MAX_NB_ATTEMPTS;
+ int nbAttempts=maxNbAttempts;
size_t repl=0, ml=0;
- U16 delta=0; // useless assignment, to remove an uninitialization warning
+ U16 delta=0; /* useless assignment, to remove an uninitialization warning */
- // HC4 match finder
+ /* HC4 match finder */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
#define REPEAT_OPTIMIZATION
#ifdef REPEAT_OPTIMIZATION
- // Detect repetitive sequences of length <= 4
- if ((U32)(ip-ref) <= 4) // potential repetition
+ /* Detect repetitive sequences of length <= 4 */
+ if ((U32)(ip-ref) <= 4) /* potential repetition */
{
- if (A32(ref) == A32(ip)) // confirmed
+ if (A32(ref) == A32(ip)) /* confirmed */
{
delta = (U16)(ip-ref);
repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
@@ -469,7 +475,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const
}
#ifdef REPEAT_OPTIMIZATION
- // Complete table
+ /* Complete table */
if (repl)
{
const BYTE* ptr = ip;
@@ -478,13 +484,13 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const
end = ip + repl - (MINMATCH-1);
while(ptr < end-delta)
{
- DELTANEXT(ptr) = delta; // Pre-Load
+ DELTANEXT(ptr) = delta; /* Pre-Load */
ptr++;
}
do
{
DELTANEXT(ptr) = delta;
- HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain
+ HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */
ptr++;
} while(ptr < end);
hc4->nextToUpdate = end;
@@ -495,16 +501,16 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const
}
-FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)
+FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
INITBASE(base,hc4->base);
const BYTE* ref;
- int nbAttempts = MAX_NB_ATTEMPTS;
+ int nbAttempts = maxNbAttempts;
int delta = (int)(ip-startLimit);
- // First Match
+ /* First Match */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
@@ -521,7 +527,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const
while (ipt<matchlimit-(STEPSIZE-1))
{
- UARCH diff = AARCH(reft) ^ AARCH(ipt);
+ size_t diff = AARCH(reft) ^ AARCH(ipt);
if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
ipt += LZ4_NbCommonBytes(diff);
goto _endCount;
@@ -532,7 +538,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const
_endCount:
reft = ref;
#else
- // Easier for code maintenance, but unfortunately slower too
+ /* Easier for code maintenance, but unfortunately slower too */
const BYTE* startt = ip;
const BYTE* reft = ref;
const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
@@ -568,26 +574,26 @@ FORCE_INLINE int LZ4HC_encodeSequence (
int length;
BYTE* token;
- // Encode Literal length
+ /* Encode Literal length */
length = (int)(*ip - *anchor);
token = (*op)++;
- if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit
+ if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */
if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
else *token = (BYTE)(length<<ML_BITS);
- // Copy Literals
+ /* Copy Literals */
LZ4_BLINDCOPY(*anchor, *op, length);
- // Encode Offset
+ /* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
- // Encode MatchLength
+ /* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
- if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit
+ if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; /* Check output limit */
if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
else *token += (BYTE)(length);
- // Prepare next loop
+ /* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
@@ -595,12 +601,14 @@ FORCE_INLINE int LZ4HC_encodeSequence (
}
+#define MAX_COMPRESSION_LEVEL 16
static int LZ4HC_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
+ int compressionLevel,
limitedOutput_directive limit
)
{
@@ -614,6 +622,7 @@ static int LZ4HC_compress_generic (
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
+ const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<<LZ4HC_DEFAULT_COMPRESSIONLEVEL;
int ml, ml2, ml3, ml0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
@@ -624,29 +633,29 @@ static int LZ4HC_compress_generic (
const BYTE* ref0;
- // Ensure blocks follow each other
+ /* Ensure blocks follow each other */
if (ip != ctx->end) return 0;
ctx->end += inputSize;
ip++;
- // Main Loop
+ /* Main Loop */
while (ip < mflimit)
{
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));
+ ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
if (!ml) { ip++; continue; }
- // saved, in case we would skip too much
+ /* saved, in case we would skip too much */
start0 = ip;
ref0 = ref;
ml0 = ml;
_Search2:
if (ip+ml < mflimit)
- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
else ml2 = ml;
- if (ml2 == ml) // No better match
+ if (ml2 == ml) /* No better match */
{
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
continue;
@@ -654,7 +663,7 @@ _Search2:
if (start0 < ip)
{
- if (start2 < ip + ml0) // empirical
+ if (start2 < ip + ml0) /* empirical */
{
ip = start0;
ref = ref0;
@@ -662,8 +671,8 @@ _Search2:
}
}
- // Here, start0==ip
- if ((start2 - ip) < 3) // First Match too small : removed
+ /* Here, start0==ip */
+ if ((start2 - ip) < 3) /* First Match too small : removed */
{
ml = ml2;
ip = start2;
@@ -672,9 +681,11 @@ _Search2:
}
_Search3:
- // Currently we have :
- // ml2 > ml1, and
- // ip1+3 <= ip2 (usually < ip1+ml1)
+ /*
+ * Currently we have :
+ * ml2 > ml1, and
+ * ip1+3 <= ip2 (usually < ip1+ml1)
+ */
if ((start2 - ip) < OPTIMAL_ML)
{
int correction;
@@ -689,26 +700,26 @@ _Search3:
ml2 -= correction;
}
}
- // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18)
+ /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
if (start2 + ml2 < mflimit)
- ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
+ ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
else ml3 = ml2;
- if (ml3 == ml2) // No better match : 2 sequences to encode
+ if (ml3 == ml2) /* No better match : 2 sequences to encode */
{
- // ip & ref are known; Now for ml
+ /* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
- // Now, encode 2 sequences
+ /* Now, encode 2 sequences */
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
ip = start2;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
continue;
}
- if (start3 < ip+ml+3) // Not enough space for match 2 : remove it
+ if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
{
- if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
+ if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
{
if (start2 < ip+ml)
{
@@ -741,8 +752,10 @@ _Search3:
goto _Search3;
}
- // OK, now we have 3 ascending matches; let's write at least the first one
- // ip & ref are known; Now for ml
+ /*
+ * OK, now we have 3 ascending matches; let's write at least the first one
+ * ip & ref are known; Now for ml
+ */
if (start2 < ip+ml)
{
if ((start2 - ip) < (int)ML_MASK)
@@ -777,80 +790,101 @@ _Search3:
}
- // Encode Last Literals
+ /* Encode Last Literals */
{
int lastRun = (int)(iend - anchor);
- if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
+ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
}
- // End
+ /* End */
return (int) (((char*)op)-dest);
}
-int LZ4_compressHC(const char* source, char* dest, int inputSize)
+int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
{
void* ctx = LZ4_createHC(source);
int result;
if (ctx==NULL) return 0;
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit);
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
LZ4_freeHC(ctx);
return result;
}
-int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
+
+int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
void* ctx = LZ4_createHC(source);
int result;
if (ctx==NULL) return 0;
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput);
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
LZ4_freeHC(ctx);
return result;
}
+int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0);
+}
-//*****************************
-// Using an external allocation
-//*****************************
+/*****************************
+ Using external allocation
+*****************************/
int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); }
-int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
+int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
{
- if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits)
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
- return LZ4HC_compress_generic (state, source, dest, inputSize, 0, noLimit);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
}
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
+{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); }
-int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+
+int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
- if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits)
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
- return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, limitedOutput);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
+
-//****************************
-// Stream functions
-//****************************
+/****************************
+ Stream functions
+****************************/
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit);
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
+}
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
{
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput);
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
}
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+}
diff --git a/lz4hc.h b/lz4hc.h
index 4fb1916..da1a159 100644
--- a/lz4hc.h
+++ b/lz4hc.h
@@ -1,157 +1,172 @@
-/*
- LZ4 HC - High Compression Mode of LZ4
- Header File
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-#pragma once
-
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-int LZ4_compressHC (const char* source, char* dest, int inputSize);
-/*
-LZ4_compressHC :
- return : the number of bytes in compressed buffer dest
- or 0 if compression fails.
- note : destination buffer must be already allocated.
- To avoid any problem, size it to handle worst cases situations (input data not compressible)
- Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
-*/
-
-int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-/*
-LZ4_compress_limitedOutput() :
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
- If it cannot achieve it, compression will stop, and result of the function will be zero.
- This function never writes outside of provided output buffer.
-
- inputSize : Max supported value is 1 GB
- maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
- return : the number of output bytes written in buffer 'dest'
- or 0 if compression fails.
-*/
-
-
-/* Note :
-Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
-*/
-
-
-//*****************************
-// Using an external allocation
-//*****************************
-int LZ4_sizeofStateHC();
-int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/*
-These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
-To know how much memory must be allocated for the compression tables, use :
-int LZ4_sizeofStateHC();
-
-Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
-
-The allocated memory can be provided to the compressions functions using 'void* state' parameter.
-LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
-They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
-*/
-
-
-//****************************
-// Streaming Functions
-//****************************
-
-void* LZ4_createHC (const char* inputBuffer);
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
-char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
-int LZ4_freeHC (void* LZ4HC_Data);
-
-/*
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
-In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
-
-void* LZ4_createHC (const char* inputBuffer);
-The result of the function is the (void*) pointer on the LZ4HC Data Structure.
-This pointer will be needed in all other functions.
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
-To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
-Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
-but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
-If next block does not begin immediately after the previous one, the compression will fail (return 0).
-
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
-==> The memory position where the next input data block must start is provided as the result of the function.
-
-Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
-
-When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
-*/
-
-int LZ4_sizeofStreamStateHC();
-int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
-
-/*
-These functions achieve the same result as :
-void* LZ4_createHC (const char* inputBuffer);
-
-They are provided here to allow the user program to allocate memory using its own routines.
-
-To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
-Note also that space must be aligned for pointers (32 or 64 bits).
-
-Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
-void* state is a pointer to the space allocated.
-It must be aligned for pointers (32 or 64 bits), and be large enough.
-The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
-The input buffer must be already allocated, and size at least 192KB.
-'inputBuffer' will also be the 'const char* source' of the first block.
-
-The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
-return value of LZ4_resetStreamStateHC() must be 0 is OK.
-Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
-*/
-
-
-#if defined (__cplusplus)
-}
-#endif
+/*
+ LZ4 HC - High Compression Mode of LZ4
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+int LZ4_compressHC (const char* source, char* dest, int inputSize);
+/*
+LZ4_compressHC :
+ return : the number of bytes in compressed buffer dest
+ or 0 if compression fails.
+ note : destination buffer must be already allocated.
+ To avoid any problem, size it to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+*/
+
+int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is 1 GB
+ maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
+ return : the number of output bytes written in buffer 'dest'
+ or 0 if compression fails.
+*/
+
+
+int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+/*
+ Same functions as above, but with programmable 'compressionLevel'.
+ Recommended values are between 4 and 9, although any value between 0 and 16 will work.
+ 'compressionLevel'==0 means use default 'compressionLevel' value.
+ Values above 16 behave the same as 16.
+ Equivalent variants exist for all other compression functions below.
+*/
+
+/* Note :
+Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+*/
+
+
+/**************************************
+ Using an external allocation
+**************************************/
+int LZ4_sizeofStateHC();
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofStateHC();
+
+Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+ Streaming Functions
+**************************************/
+void* LZ4_createHC (const char* inputBuffer);
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+int LZ4_freeHC (void* LZ4HC_Data);
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
+
+void* LZ4_createHC (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4HC Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
+Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
+but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
+*/
+
+int LZ4_sizeofStreamStateHC();
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_createHC (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
+Note also that space must be aligned for pointers (32 or 64 bits).
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned for pointers (32 or 64 bits), and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamStateHC() must be 0 is OK.
+Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
+*/
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/programs/Makefile b/programs/Makefile
index 5f86d3c..98ed185 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ################################################################
-RELEASE=r112
+RELEASE=r113
DESTDIR=
PREFIX=/usr
CC=gcc
@@ -53,13 +53,13 @@ default: lz4 lz4c
all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32
-lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c
+lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT)
-lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c
+lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) -O3 $(CFLAGS) $^ -o $@$(EXT)
-lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c
+lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c
$(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT)
fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c
diff --git a/programs/bench.c b/programs/bench.c
index df3c44a..ea6a1ca 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -59,9 +59,10 @@
#endif
#include "lz4.h"
-#define COMPRESSOR0 LZ4_compress
+#define COMPRESSOR0 LZ4_compress_local
+static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); }
#include "lz4hc.h"
-#define COMPRESSOR1 LZ4_compressHC
+#define COMPRESSOR1 LZ4_compressHC2
#define DEFAULTCOMPRESSOR COMPRESSOR0
#include "xxhash.h"
@@ -128,7 +129,7 @@ struct chunkParameters
struct compressionParameters
{
- int (*compressionFunction)(const char*, char*, int);
+ int (*compressionFunction)(const char*, char*, int, int);
int (*decompressionFunction)(const char*, char*, int);
};
@@ -139,7 +140,6 @@ struct compressionParameters
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
-
//**************************************
// Benchmark Parameters
//**************************************
@@ -254,7 +254,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
// Init
- if (cLevel <3) cfunctionId = 0; else cfunctionId = 1;
+ if (cLevel <= 3) cfunctionId = 0; else cfunctionId = 1;
switch (cfunctionId)
{
#ifdef COMPRESSOR0
@@ -377,7 +377,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
- chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
+ chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 4513ebe..7b06cbe 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -1,7 +1,6 @@
/*
fuzzer.c - Fuzzer test tool for LZ4
- Copyright (C) Yann Collet - Andrew Mahone 2012-2013
- Code started by Andrew Mahone, modified by Yann Collet
+ Copyright (C) Yann Collet - Andrew Mahone 2012-2014
GPL v2 License
This program is free software; you can redistribute it and/or modify
@@ -26,7 +25,7 @@
//**************************************
// Remove Visual warning messages
//**************************************
-#define _CRT_SECURE_NO_WARNINGS // fgets
+#define _CRT_SECURE_NO_WARNINGS // fgets
//**************************************
diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index f69c8a0..26e0111 100644
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -63,6 +63,7 @@
#include "lz4hc.h"
#include "xxhash.h"
#include "bench.h"
+#include "lz4io.h"
//****************************
@@ -108,7 +109,7 @@
//****************************
#define COMPRESSOR_NAME "LZ4 Compression CLI"
#ifndef LZ4_VERSION
-# define LZ4_VERSION "v1.1.0"
+# define LZ4_VERSION "v1.1.3"
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
@@ -118,25 +119,7 @@
#define MB *(1U<<20)
#define GB *(1U<<30)
-#define _1BIT 0x01
-#define _2BITS 0x03
-#define _3BITS 0x07
-#define _4BITS 0x0F
-#define _8BITS 0xFF
-
-#define MAGICNUMBER_SIZE 4
-#define LZ4S_MAGICNUMBER 0x184D2204
-#define LZ4S_SKIPPABLE0 0x184D2A50
-#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
-#define LEGACY_MAGICNUMBER 0x184C2102
-
-#define CACHELINE 64
-#define LEGACY_BLOCKSIZE (8 MB)
-#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
-#define LZ4S_BLOCKSIZEID_DEFAULT 7
-#define LZ4S_CHECKSUM_SEED 0
-#define LZ4S_EOS 0
-#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
+#define LZ4_BLOCKSIZEID_DEFAULT 7
//**************************************
@@ -156,28 +139,10 @@ static const int one = 1;
//**************************************
-// Special input/output
-//**************************************
-#define NULL_OUTPUT "null"
-char stdinmark[] = "stdin";
-char stdoutmark[] = "stdout";
-#ifdef _WIN32
-char nulmark[] = "nul";
-#else
-char nulmark[] = "/dev/null";
-#endif
-
-
-//**************************************
// Local Parameters
//**************************************
static char* programName;
static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information
-static int overwrite = 0;
-static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
-static int blockChecksum = 0;
-static int streamChecksum = 1;
-static int blockIndependence = 1;
//**************************************
@@ -201,8 +166,9 @@ static int blockIndependence = 1;
#define EXTENDED_ARGUMENTS
#define EXTENDED_HELP
#define EXTENDED_FORMAT
-#define DEFAULT_COMPRESSOR compress_file
-#define DEFAULT_DECOMPRESSOR decodeLZ4S
+#define DEFAULT_COMPRESSOR LZ4IO_compressFilename
+#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
+int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel);
//****************************
@@ -314,705 +280,6 @@ int badusage()
}
-static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
-static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }
-static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
-
-
-int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
-{
-
- if (!strcmp (input_filename, stdinmark))
- {
- DISPLAYLEVEL(4,"Using stdin for input\n");
- *pfinput = stdin;
- SET_BINARY_MODE(stdin);
- }
- else
- {
- *pfinput = fopen(input_filename, "rb");
- }
-
- if (!strcmp (output_filename, stdoutmark))
- {
- DISPLAYLEVEL(4,"Using stdout for output\n");
- *pfoutput = stdout;
- SET_BINARY_MODE(stdout);
- }
- else
- {
- // Check if destination file already exists
- *pfoutput=0;
- if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );
- if (*pfoutput!=0)
- {
- fclose(*pfoutput);
- if (!overwrite)
- {
- char ch;
- DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
- DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
- if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible
- ch = (char)getchar();
- if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
- }
- }
- *pfoutput = fopen( output_filename, "wb" );
- }
-
- if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
- if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
-
- return 0;
-}
-
-
-
-int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel)
-{
- int (*compressionFunction)(const char*, char*, int);
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
- char* in_buff;
- char* out_buff;
- FILE* finput;
- FILE* foutput;
- int displayLevel = (compressionlevel>0);
- clock_t start, end;
- size_t sizeCheck;
-
-
- // Init
- if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
- start = clock();
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
-
- // Allocate Memory
- in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
- if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
-
- // Write Archive Header
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);
- sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
- if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");
-
- // Main Loop
- while (1)
- {
- unsigned int outSize;
- // Read Block
- int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
- if( inSize<=0 ) break;
- filesize += inSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
-
- // Compress Block
- outSize = compressionFunction(in_buff, out_buff+4, inSize);
- compressedfilesize += outSize+4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
- if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");
- }
-
- // Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close & Free
- free(in_buff);
- free(out_buff);
- fclose(finput);
- fclose(foutput);
-
- return 0;
-}
-
-
-int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
-{
- void* (*initFunction) (const char*);
- int (*compressionFunction)(void*, const char*, char*, int, int);
- char* (*translateFunction) (void*);
- int (*freeFunction) (void*);
- void* ctx;
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = 0;
- unsigned int checkbits;
- char* in_buff, *in_start, *in_end;
- char* out_buff;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
- unsigned int blockSize, inputBufferSize;
- size_t sizeCheck, header_size;
- void* streamChecksumState=NULL;
-
-
- // Init
- start = clock();
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
- if (compressionlevel>=3)
- {
- initFunction = LZ4_createHC;
- compressionFunction = LZ4_compressHC_limitedOutput_continue;
- translateFunction = LZ4_slideInputBufferHC;
- freeFunction = LZ4_freeHC;
- }
- else
- {
- initFunction = LZ4_create;
- compressionFunction = LZ4_compress_limitedOutput_continue;
- translateFunction = LZ4_slideInputBuffer;
- freeFunction = LZ4_free;
- }
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
-
- // Allocate Memory
- inputBufferSize = blockSize + 64 KB;
- if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
- in_buff = (char*)malloc(inputBufferSize);
- out_buff = (char*)malloc(blockSize+CACHELINE);
- if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
- in_start = in_buff; in_end = in_buff + inputBufferSize;
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
- ctx = initFunction(in_buff);
-
- // Write Archive Header
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
- *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')
- *(out_buff+4) |= (blockIndependence & _1BIT) << 5;
- *(out_buff+4) |= (blockChecksum & _1BIT) << 4;
- *(out_buff+4) |= (streamChecksum & _1BIT) << 2;
- *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);
- checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
- *(out_buff+6) = (unsigned char) checkbits;
- header_size = 7;
- sizeCheck = fwrite(out_buff, 1, header_size, foutput);
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
- compressedfilesize += header_size;
-
- // Main Loop
- while (1)
- {
- unsigned int outSize;
- unsigned int inSize;
- // Read Block
- if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
- inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
- if( inSize==0 ) break; // No more input : end of compression
- filesize += inSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
- if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
-
- // Compress Block
- outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
- if (blockChecksum) compressedfilesize+=4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- if (outSize > 0)
- {
- int sizeToWrite;
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
- }
- sizeToWrite = 4 + outSize + (4*blockChecksum);
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
-
- }
- else // Copy Original
- {
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
- sizeCheck = fwrite(in_start, 1, inSize, foutput);
- if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
- }
- }
- in_start += inSize;
- }
-
- // End of Stream mark
- * (unsigned int*) out_buff = LZ4S_EOS;
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
- compressedfilesize += 4;
- if (streamChecksum)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
- compressedfilesize += 4;
- }
-
- // Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close & Free
- freeFunction(ctx);
- free(in_buff);
- free(out_buff);
- fclose(finput);
- fclose(foutput);
-
- return 0;
-}
-
-
-int compress_file(char* input_filename, char* output_filename, int compressionlevel)
-{
- int (*compressionFunction)(const char*, char*, int, int);
- unsigned long long filesize = 0;
- unsigned long long compressedfilesize = 0;
- unsigned int checkbits;
- char* in_buff;
- char* out_buff;
- char* headerBuffer;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
- int blockSize;
- size_t sizeCheck, header_size, readSize;
- void* streamChecksumState=NULL;
-
- // Branch out
- if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);
-
- // Init
- start = clock();
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
- if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput;
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
-
- // Allocate Memory
- in_buff = (char*)malloc(blockSize);
- out_buff = (char*)malloc(blockSize+CACHELINE);
- headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE);
- if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory");
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
-
- // Write Archive Header
- *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
- *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01')
- *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5;
- *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4;
- *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2;
- *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4);
- checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED);
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
- *(headerBuffer+6) = (unsigned char) checkbits;
- header_size = 7;
-
- // Write header
- sizeCheck = fwrite(headerBuffer, 1, header_size, foutput);
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
- compressedfilesize += header_size;
-
- // read first block
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
-
- // Main Loop
- while (readSize>0)
- {
- unsigned int outSize;
-
- filesize += readSize;
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
- if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize);
-
- // Compress Block
- outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1);
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
- if (blockChecksum) compressedfilesize+=4;
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
-
- // Write Block
- if (outSize > 0)
- {
- int sizeToWrite;
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
- }
- sizeToWrite = 4 + outSize + (4*blockChecksum);
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
- }
- else // Copy Original Uncompressed
- {
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
- sizeCheck = fwrite(in_buff, 1, readSize, foutput);
- if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block");
- if (blockChecksum)
- {
- unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
- }
- }
-
- // Read next block
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
- }
-
- // End of Stream mark
- * (unsigned int*) out_buff = LZ4S_EOS;
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
- compressedfilesize += 4;
- if (streamChecksum)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
- sizeCheck = fwrite(out_buff, 1, 4, foutput);
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
- compressedfilesize += 4;
- }
-
- // Close & Free
- free(in_buff);
- free(out_buff);
- free(headerBuffer);
- fclose(finput);
- fclose(foutput);
-
- // Final Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- return 0;
-}
-
-
-unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
-{
- unsigned long long filesize = 0;
- char* in_buff;
- char* out_buff;
- unsigned int blockSize;
-
-
- // Allocate Memory
- in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
- out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
-
- // Main Loop
- while (1)
- {
- int decodeSize;
- size_t sizeCheck;
-
- // Block Size
- sizeCheck = fread(&blockSize, 1, 4, finput);
- if (sizeCheck==0) break; // Nothing to read : file read is completed
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian
- if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
- { // Cannot read next block : maybe new stream ?
- fseek(finput, -4, SEEK_CUR);
- break;
- }
-
- // Read Block
- sizeCheck = fread(in_buff, 1, blockSize, finput);
-
- // Decode Block
- decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
- if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
- filesize += decodeSize;
-
- // Write Block
- sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
- if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
- }
-
- // Free
- free(in_buff);
- free(out_buff);
-
- return filesize;
-}
-
-
-unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
-{
- unsigned long long filesize = 0;
- char* in_buff;
- char* out_buff, *out_start, *out_end;
- unsigned char descriptor[LZ4S_MAXHEADERSIZE];
- size_t nbReadBytes;
- int decodedBytes=0;
- unsigned int maxBlockSize;
- size_t sizeCheck;
- int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
- void* streamChecksumState=NULL;
- int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
- unsigned int prefix64k = 0;
-
- // Decode stream descriptor
- nbReadBytes = fread(descriptor, 1, 3, finput);
- if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");
- {
- int version = (descriptor[0] >> 6) & _2BITS;
- int streamSize = (descriptor[0] >> 3) & _1BIT;
- int reserved1 = (descriptor[0] >> 1) & _1BIT;
- int dictionary = (descriptor[0] >> 0) & _1BIT;
-
- int reserved2 = (descriptor[1] >> 7) & _1BIT;
- int blockSizeId = (descriptor[1] >> 4) & _3BITS;
- int reserved3 = (descriptor[1] >> 0) & _4BITS;
- int checkBits = (descriptor[2] >> 0) & _8BITS;
- int checkBits_xxh32;
-
- blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;
- blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;
- streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;
-
- if (version != 1) EXM_THROW(62, "Wrong version number");
- if (streamSize == 1) EXM_THROW(64, "Does not support stream size");
- if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");
- if (dictionary == 1) EXM_THROW(66, "Does not support dictionary");
- if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");
- if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size");
- if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");
- maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);
- // Checkbits verification
- descriptor[1] &= 0xF0;
- checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);
- checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);
- if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
- }
-
- if (!blockIndependenceFlag)
- {
- decompressionFunction = LZ4_decompress_safe_withPrefix64k;
- prefix64k = 64 KB;
- }
-
- // Allocate Memory
- {
- unsigned int outbuffSize = prefix64k+maxBlockSize;
- in_buff = (char*)malloc(maxBlockSize);
- if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
- out_buff = (char*)malloc(outbuffSize);
- out_end = out_buff + outbuffSize;
- out_start = out_buff + prefix64k;
- if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
- }
- if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
-
- // Main Loop
- while (1)
- {
- unsigned int blockSize, uncompressedFlag;
-
- // Block Size
- nbReadBytes = fread(&blockSize, 1, 4, finput);
- if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");
- if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian
- uncompressedFlag = blockSize >> 31;
- blockSize &= 0x7FFFFFFF;
- if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");
-
- // Read Block
- nbReadBytes = fread(in_buff, 1, blockSize, finput);
- if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );
-
- // Check Block
- if (blockChecksumFlag)
- {
- unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);
- unsigned int readChecksum;
- sizeCheck = fread(&readChecksum, 1, 4, finput);
- if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");
- }
-
- if (uncompressedFlag)
- {
- // Write uncompressed Block
- sizeCheck = fwrite(in_buff, 1, blockSize, foutput);
- if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");
- filesize += blockSize;
- if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
- if (!blockIndependenceFlag)
- {
- if (blockSize >= prefix64k)
- {
- memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
- out_start = out_buff + prefix64k;
- continue;
- }
- else
- {
- memcpy(out_start, in_buff, blockSize);
- decodedBytes = blockSize;
- }
- }
- }
- else
- {
- // Decode Block
- decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
- if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
- filesize += decodedBytes;
- if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
-
- // Write Block
- sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
- if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
- }
-
- if (!blockIndependenceFlag)
- {
- out_start += decodedBytes;
- if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
- {
- memcpy(out_buff, out_start - prefix64k, prefix64k);
- out_start = out_buff + prefix64k;
- }
- }
- }
-
- // Stream Checksum
- if (streamChecksumFlag)
- {
- unsigned int checksum = XXH32_digest(streamChecksumState);
- unsigned int readChecksum;
- sizeCheck = fread(&readChecksum, 1, 4, finput);
- if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
- }
-
- // Free
- free(in_buff);
- free(out_buff);
-
- return filesize;
-}
-
-
-unsigned long long selectDecoder( FILE* finput, FILE* foutput)
-{
- unsigned int magicNumber, size;
- int errorNb;
- size_t nbReadBytes;
-
- // Check Archive Header
- nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);
- if (nbReadBytes==0) return 0; // EOF
- if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");
- magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format
- if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers
-
- switch(magicNumber)
- {
- case LZ4S_MAGICNUMBER:
- return DEFAULT_DECOMPRESSOR(finput, foutput);
- case LEGACY_MAGICNUMBER:
- DISPLAYLEVEL(4, "Detected : Legacy format \n");
- return decodeLegacyStream(finput, foutput);
- case LZ4S_SKIPPABLE0:
- DISPLAYLEVEL(4, "Skipping detected skippable area \n");
- nbReadBytes = fread(&size, 1, 4, finput);
- if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
- size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format
- errorNb = fseek(finput, size, SEEK_CUR);
- if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
- return selectDecoder(finput, foutput);
- EXTENDED_FORMAT;
- default:
- if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream
- DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
- return 0;
- }
-}
-
-
-int decodeFile(char* input_filename, char* output_filename)
-{
- unsigned long long filesize = 0, decodedSize=0;
- FILE* finput;
- FILE* foutput;
- clock_t start, end;
-
-
- // Init
- start = clock();
- get_fileHandle(input_filename, output_filename, &finput, &foutput);
-
- // Loop over multiple streams
- do
- {
- decodedSize = selectDecoder(finput, foutput);
- filesize += decodedSize;
- } while (decodedSize);
-
- // Final Status
- end = clock();
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
- {
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
- }
-
- // Close
- fclose(finput);
- fclose(foutput);
-
- // Error status = OK
- return 0;
-}
-
-
void waitEnter()
{
DISPLAY("Press enter to continue...\n");
@@ -1036,9 +303,12 @@ int main(int argc, char** argv)
char* dynNameSpace=0;
char nullOutput[] = NULL_OUTPUT;
char extension[] = LZ4_EXTENSION;
+ int blockSize;
// Init
programName = argv[0];
+ LZ4IO_setOverwrite(0);
+ blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
for(i=1; i<argc; i++)
{
@@ -1065,7 +335,7 @@ int main(int argc, char** argv)
if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression)
if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression)
if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression)
- if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission)
+ if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } // -y (answer 'yes' to overwrite permission)
if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
#endif // DISABLE_LZ4C_LEGACY_OPTIONS
@@ -1089,7 +359,9 @@ int main(int argc, char** argv)
case '6':
case '7':
case '8':
- case '9': cLevel=*argument -'0'; break;
+ case '9':
+ case 'A': /* non documented (hidden) */
+ cLevel=*argument -'0'; break;
// Use Legacy format (for Linux kernel compression)
case 'l': legacy_format=1; break;
@@ -1104,7 +376,7 @@ int main(int argc, char** argv)
case 't': decode=1; output_filename=nulmark; break;
// Overwrite
- case 'f': overwrite=1; break;
+ case 'f': LZ4IO_setOverwrite(1); break;
// Verbose mode
case 'v': displayLevel=4; break;
@@ -1130,12 +402,12 @@ int main(int argc, char** argv)
int B = argument[1] - '0';
int S = 1 << (8 + 2*B);
BMK_SetBlocksize(S);
- blockSizeId = B;
+ LZ4IO_setBlockSizeID(B);
argument++;
break;
}
- case 'D': blockIndependence = 0, argument++; break;
- case 'X': blockChecksum = 1, argument ++; break;
+ case 'D': LZ4IO_setBlockMode(independentBlocks); argument++; break;
+ case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break;
default : exitBlockProperties=1;
}
if (exitBlockProperties) break;
@@ -1143,7 +415,7 @@ int main(int argc, char** argv)
break;
// Modify Stream properties
- case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); }
+ case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); }
// Benchmark
case 'b': bench=1; break;
@@ -1183,7 +455,7 @@ int main(int argc, char** argv)
}
DISPLAYLEVEL(3, WELCOME_MESSAGE);
- DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2)));
+ DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
// No input filename ==> use stdin
if(!input_filename) { input_filename=stdinmark; }
@@ -1235,15 +507,16 @@ int main(int argc, char** argv)
if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
- // Decompress input if selected
- if (decode) decodeFile(input_filename, output_filename);
+ // IO Stream/File
+ LZ4IO_setNotificationLevel(displayLevel);
+ if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename);
else
// compression is default action
{
if (legacy_format)
{
DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n");
- legacy_compress_file(input_filename, output_filename, cLevel);
+ LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel);
}
else
{
diff --git a/programs/lz4io.c b/programs/lz4io.c
new file mode 100644
index 0000000..ae8b776
--- /dev/null
+++ b/programs/lz4io.c
@@ -0,0 +1,948 @@
+/*
+ LZ4io.c - LZ4 File/Stream Interface
+ Copyright (C) Yann Collet 2011-2013
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+/*
+ Note : this is stand-alone program.
+ It is not part of LZ4 compression library, it is a user code of the LZ4 library.
+ - The license of LZ4 library is BSD.
+ - The license of xxHash library is BSD.
+ - The license of this source file is GPLv2.
+*/
+
+//**************************************
+// Compiler Options
+//**************************************
+#ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# define _CRT_SECURE_NO_WARNINGS
+# define _CRT_SECURE_NO_DEPRECATE // VS2005
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+#else
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+#endif
+
+#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix
+#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix
+
+
+//****************************
+// Includes
+//****************************
+#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout
+#include <stdlib.h> // malloc
+#include <string.h> // strcmp, strlen
+#include <time.h> // clock
+#include "lz4io.h"
+#include "lz4.h"
+#include "lz4hc.h"
+#include "xxhash.h"
+
+
+//****************************
+// OS-specific Includes
+//****************************
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+# include <fcntl.h> // _O_BINARY
+# include <io.h> // _setmode, _isatty
+# ifdef __MINGW32__
+ int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h>
+# endif
+# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+# include <unistd.h> // isatty
+# define SET_BINARY_MODE(file)
+# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#endif
+
+
+//**************************************
+// Compiler-specific functions
+//**************************************
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#if defined(_MSC_VER) // Visual Studio
+# define swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+# define swap32 __builtin_bswap32
+#else
+ static inline unsigned int swap32(unsigned int x)
+ {
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+ }
+#endif
+
+
+//****************************
+// Constants
+//****************************
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+#define _1BIT 0x01
+#define _2BITS 0x03
+#define _3BITS 0x07
+#define _4BITS 0x0F
+#define _8BITS 0xFF
+
+#define MAGICNUMBER_SIZE 4
+#define LZ4S_MAGICNUMBER 0x184D2204
+#define LZ4S_SKIPPABLE0 0x184D2A50
+#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
+#define LEGACY_MAGICNUMBER 0x184C2102
+
+#define CACHELINE 64
+#define LEGACY_BLOCKSIZE (8 MB)
+#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
+#define LZ4S_BLOCKSIZEID_DEFAULT 7
+#define LZ4S_CHECKSUM_SEED 0
+#define LZ4S_EOS 0
+#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
+
+
+//**************************************
+// Architecture Macros
+//**************************************
+static const int one = 1;
+#define CPU_LITTLE_ENDIAN (*(char*)(&one))
+#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)
+#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i))
+
+
+//**************************************
+// Macros
+//**************************************
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+
+//**************************************
+// Local Parameters
+//**************************************
+static int displayLevel = 0; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information
+static int overwrite = 1;
+static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
+static int blockChecksum = 0;
+static int streamChecksum = 1;
+static int blockIndependence = 1;
+
+static const int minBlockSizeID = 4;
+static const int maxBlockSizeID = 7;
+
+//**************************************
+// Exceptions
+//**************************************
+#define DEBUG 0
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+
+//**************************************
+// Version modifiers
+//**************************************
+#define EXTENDED_ARGUMENTS
+#define EXTENDED_HELP
+#define EXTENDED_FORMAT
+#define DEFAULT_COMPRESSOR compress_file
+#define DEFAULT_DECOMPRESSOR decodeLZ4S
+
+
+/* ************************************************** */
+/* ****************** Parameters ******************** */
+/* ************************************************** */
+
+/* Default setting : overwrite = 1; return : overwrite mode (0/1) */
+int LZ4IO_setOverwrite(int yes)
+{
+ overwrite = (yes!=0);
+ return overwrite;
+}
+
+/* blockSizeID : valid values : 4-5-6-7 */
+int LZ4IO_setBlockSizeID(int bsid)
+{
+ static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
+ if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1;
+ blockSizeId = bsid;
+ return blockSizeTable[blockSizeId-minBlockSizeID];
+}
+
+
+int LZ4IO_setBlockMode(blockMode_t blockMode)
+{
+ blockIndependence = (blockMode == independentBlocks);
+ return blockIndependence;
+}
+
+
+/* Default setting : no checksum */
+int LZ4IO_setBlockChecksumMode(int xxhash)
+{
+ blockChecksum = (xxhash != 0);
+ return blockChecksum;
+}
+
+
+/* Default setting : checksum enabled */
+int LZ4IO_setStreamChecksumMode(int xxhash)
+{
+ streamChecksum = (xxhash != 0);
+ return streamChecksum;
+}
+
+
+/* Default setting : 0 (no notification) */
+int LZ4IO_setNotificationLevel(int level)
+{
+ displayLevel = level;
+ return displayLevel;
+}
+
+
+
+/* ************************************************************************ */
+/* ********************** LZ4 File / Stream compression ******************* */
+/* ************************************************************************ */
+
+static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
+static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }
+static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
+
+
+static int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
+{
+
+ if (!strcmp (input_filename, stdinmark))
+ {
+ DISPLAYLEVEL(4,"Using stdin for input\n");
+ *pfinput = stdin;
+ SET_BINARY_MODE(stdin);
+ }
+ else
+ {
+ *pfinput = fopen(input_filename, "rb");
+ }
+
+ if (!strcmp (output_filename, stdoutmark))
+ {
+ DISPLAYLEVEL(4,"Using stdout for output\n");
+ *pfoutput = stdout;
+ SET_BINARY_MODE(stdout);
+ }
+ else
+ {
+ // Check if destination file already exists
+ *pfoutput=0;
+ if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );
+ if (*pfoutput!=0)
+ {
+ fclose(*pfoutput);
+ if (!overwrite)
+ {
+ char ch;
+ DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
+ DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
+ if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible
+ ch = (char)getchar();
+ if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
+ }
+ }
+ *pfoutput = fopen( output_filename, "wb" );
+ }
+
+ if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
+ if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
+
+ return 0;
+}
+
+
+// LZ4IO_compressFilename_Legacy : This function is "hidden" (not published in .h)
+// Its purpose is to generate compressed streams using the old 'legacy' format
+int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel)
+{
+ int (*compressionFunction)(const char*, char*, int);
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
+ char* in_buff;
+ char* out_buff;
+ FILE* finput;
+ FILE* foutput;
+ int displayLevel = (compressionlevel>0);
+ clock_t start, end;
+ size_t sizeCheck;
+
+
+ // Init
+ if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
+ start = clock();
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
+
+ // Allocate Memory
+ in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
+ out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
+
+ // Write Archive Header
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);
+ sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
+ if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int outSize;
+ // Read Block
+ int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
+ if( inSize<=0 ) break;
+ filesize += inSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+
+ // Compress Block
+ outSize = compressionFunction(in_buff, out_buff+4, inSize);
+ compressedfilesize += outSize+4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
+ if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");
+ }
+
+ // Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close & Free
+ free(in_buff);
+ free(out_buff);
+ fclose(finput);
+ fclose(foutput);
+
+ return 0;
+}
+
+
+static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
+{
+ void* (*initFunction) (const char*);
+ int (*compressionFunction)(void*, const char*, char*, int, int);
+ char* (*translateFunction) (void*);
+ int (*freeFunction) (void*);
+ void* ctx;
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = 0;
+ unsigned int checkbits;
+ char* in_buff, *in_start, *in_end;
+ char* out_buff;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+ unsigned int blockSize, inputBufferSize;
+ size_t sizeCheck, header_size;
+ void* streamChecksumState=NULL;
+
+
+ // Init
+ start = clock();
+ if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
+ if (compressionlevel>=3)
+ {
+ initFunction = LZ4_createHC;
+ compressionFunction = LZ4_compressHC_limitedOutput_continue;
+ translateFunction = LZ4_slideInputBufferHC;
+ freeFunction = LZ4_freeHC;
+ }
+ else
+ {
+ initFunction = LZ4_create;
+ compressionFunction = LZ4_compress_limitedOutput_continue;
+ translateFunction = LZ4_slideInputBuffer;
+ freeFunction = LZ4_free;
+ }
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
+
+ // Allocate Memory
+ inputBufferSize = blockSize + 64 KB;
+ if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
+ in_buff = (char*)malloc(inputBufferSize);
+ out_buff = (char*)malloc(blockSize+CACHELINE);
+ if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
+ in_start = in_buff; in_end = in_buff + inputBufferSize;
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+ ctx = initFunction(in_buff);
+
+ // Write Archive Header
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;
+ *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);
+ checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
+ *(out_buff+6) = (unsigned char) checkbits;
+ header_size = 7;
+ sizeCheck = fwrite(out_buff, 1, header_size, foutput);
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
+ compressedfilesize += header_size;
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int outSize;
+ unsigned int inSize;
+ // Read Block
+ if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
+ inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
+ if( inSize==0 ) break; // No more input : end of compression
+ filesize += inSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+ if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
+
+ // Compress Block
+ outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
+ if (blockChecksum) compressedfilesize+=4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ if (outSize > 0)
+ {
+ int sizeToWrite;
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
+ }
+ sizeToWrite = 4 + outSize + (4*blockChecksum);
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
+
+ }
+ else // Copy Original
+ {
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
+ sizeCheck = fwrite(in_start, 1, inSize, foutput);
+ if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
+ }
+ }
+ in_start += inSize;
+ }
+
+ // End of Stream mark
+ * (unsigned int*) out_buff = LZ4S_EOS;
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
+ compressedfilesize += 4;
+ if (streamChecksum)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
+ compressedfilesize += 4;
+ }
+
+ // Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close & Free
+ freeFunction(ctx);
+ free(in_buff);
+ free(out_buff);
+ fclose(finput);
+ fclose(foutput);
+
+ return 0;
+}
+
+
+FORCE_INLINE int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel)
+{ (void)clevel; return LZ4_compress_limitedOutput(src, dst, size, maxOut); }
+
+int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel)
+{
+ int (*compressionFunction)(const char*, char*, int, int, int);
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = 0;
+ unsigned int checkbits;
+ char* in_buff;
+ char* out_buff;
+ char* headerBuffer;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+ int blockSize;
+ size_t sizeCheck, header_size, readSize;
+ void* streamChecksumState=NULL;
+
+ // Branch out
+ if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionLevel);
+
+ // Init
+ start = clock();
+ if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
+ if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local;
+ else { compressionFunction = LZ4_compressHC2_limitedOutput; }
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
+
+ // Allocate Memory
+ in_buff = (char*)malloc(blockSize);
+ out_buff = (char*)malloc(blockSize+CACHELINE);
+ headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE);
+ if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory");
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+
+ // Write Archive Header
+ *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
+ *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01')
+ *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5;
+ *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4;
+ *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2;
+ *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4);
+ checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED);
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
+ *(headerBuffer+6) = (unsigned char) checkbits;
+ header_size = 7;
+
+ // Write header
+ sizeCheck = fwrite(headerBuffer, 1, header_size, foutput);
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
+ compressedfilesize += header_size;
+
+ // read first block
+ readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
+
+ // Main Loop
+ while (readSize>0)
+ {
+ unsigned int outSize;
+
+ filesize += readSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+ if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize);
+
+ // Compress Block
+ outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel);
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
+ if (blockChecksum) compressedfilesize+=4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ if (outSize > 0)
+ {
+ int sizeToWrite;
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
+ }
+ sizeToWrite = 4 + outSize + (4*blockChecksum);
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
+ }
+ else // Copy Original Uncompressed
+ {
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
+ sizeCheck = fwrite(in_buff, 1, readSize, foutput);
+ if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block");
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
+ }
+ }
+
+ // Read next block
+ readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
+ }
+
+ // End of Stream mark
+ * (unsigned int*) out_buff = LZ4S_EOS;
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
+ compressedfilesize += 4;
+ if (streamChecksum)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
+ compressedfilesize += 4;
+ }
+
+ // Close & Free
+ free(in_buff);
+ free(out_buff);
+ free(headerBuffer);
+ fclose(finput);
+ fclose(foutput);
+
+ // Final Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ return 0;
+}
+
+
+/* ********************************************************************* */
+/* ********************** LZ4 File / Stream decoding ******************* */
+/* ********************************************************************* */
+
+static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
+{
+ unsigned long long filesize = 0;
+ char* in_buff;
+ char* out_buff;
+ unsigned int blockSize;
+
+
+ // Allocate Memory
+ in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
+ if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
+
+ // Main Loop
+ while (1)
+ {
+ int decodeSize;
+ size_t sizeCheck;
+
+ // Block Size
+ sizeCheck = fread(&blockSize, 1, 4, finput);
+ if (sizeCheck==0) break; // Nothing to read : file read is completed
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian
+ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
+ { // Cannot read next block : maybe new stream ?
+ fseek(finput, -4, SEEK_CUR);
+ break;
+ }
+
+ // Read Block
+ sizeCheck = fread(in_buff, 1, blockSize, finput);
+
+ // Decode Block
+ decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
+ if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
+ filesize += decodeSize;
+
+ // Write Block
+ sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
+ if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
+ }
+
+ // Free
+ free(in_buff);
+ free(out_buff);
+
+ return filesize;
+}
+
+
+static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
+{
+ unsigned long long filesize = 0;
+ char* in_buff;
+ char* out_buff, *out_start, *out_end;
+ unsigned char descriptor[LZ4S_MAXHEADERSIZE];
+ size_t nbReadBytes;
+ int decodedBytes=0;
+ unsigned int maxBlockSize;
+ size_t sizeCheck;
+ int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
+ void* streamChecksumState=NULL;
+ int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
+ unsigned int prefix64k = 0;
+
+ // Decode stream descriptor
+ nbReadBytes = fread(descriptor, 1, 3, finput);
+ if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");
+ {
+ int version = (descriptor[0] >> 6) & _2BITS;
+ int streamSize = (descriptor[0] >> 3) & _1BIT;
+ int reserved1 = (descriptor[0] >> 1) & _1BIT;
+ int dictionary = (descriptor[0] >> 0) & _1BIT;
+
+ int reserved2 = (descriptor[1] >> 7) & _1BIT;
+ int blockSizeId = (descriptor[1] >> 4) & _3BITS;
+ int reserved3 = (descriptor[1] >> 0) & _4BITS;
+ int checkBits = (descriptor[2] >> 0) & _8BITS;
+ int checkBits_xxh32;
+
+ blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;
+ blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;
+ streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;
+
+ if (version != 1) EXM_THROW(62, "Wrong version number");
+ if (streamSize == 1) EXM_THROW(64, "Does not support stream size");
+ if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");
+ if (dictionary == 1) EXM_THROW(66, "Does not support dictionary");
+ if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");
+ if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size");
+ if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");
+ maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);
+ // Checkbits verification
+ descriptor[1] &= 0xF0;
+ checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);
+ checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);
+ if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
+ }
+
+ if (!blockIndependenceFlag)
+ {
+ decompressionFunction = LZ4_decompress_safe_withPrefix64k;
+ prefix64k = 64 KB;
+ }
+
+ // Allocate Memory
+ {
+ unsigned int outbuffSize = prefix64k+maxBlockSize;
+ in_buff = (char*)malloc(maxBlockSize);
+ if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
+ out_buff = (char*)malloc(outbuffSize);
+ out_end = out_buff + outbuffSize;
+ out_start = out_buff + prefix64k;
+ if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
+ }
+ if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int blockSize, uncompressedFlag;
+
+ // Block Size
+ nbReadBytes = fread(&blockSize, 1, 4, finput);
+ if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");
+ if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian
+ uncompressedFlag = blockSize >> 31;
+ blockSize &= 0x7FFFFFFF;
+ if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");
+
+ // Read Block
+ nbReadBytes = fread(in_buff, 1, blockSize, finput);
+ if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );
+
+ // Check Block
+ if (blockChecksumFlag)
+ {
+ unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);
+ unsigned int readChecksum;
+ sizeCheck = fread(&readChecksum, 1, 4, finput);
+ if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");
+ }
+
+ if (uncompressedFlag)
+ {
+ // Write uncompressed Block
+ sizeCheck = fwrite(in_buff, 1, blockSize, foutput);
+ if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");
+ filesize += blockSize;
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
+ if (!blockIndependenceFlag)
+ {
+ if (blockSize >= prefix64k)
+ {
+ memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
+ out_start = out_buff + prefix64k;
+ continue;
+ }
+ else
+ {
+ memcpy(out_start, in_buff, blockSize);
+ decodedBytes = blockSize;
+ }
+ }
+ }
+ else
+ {
+ // Decode Block
+ decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
+ if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
+ filesize += decodedBytes;
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
+
+ // Write Block
+ sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
+ if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
+ }
+
+ if (!blockIndependenceFlag)
+ {
+ out_start += decodedBytes;
+ if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
+ {
+ memcpy(out_buff, out_start - prefix64k, prefix64k);
+ out_start = out_buff + prefix64k;
+ }
+ }
+ }
+
+ // Stream Checksum
+ if (streamChecksumFlag)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ unsigned int readChecksum;
+ sizeCheck = fread(&readChecksum, 1, 4, finput);
+ if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
+ }
+
+ // Free
+ free(in_buff);
+ free(out_buff);
+
+ return filesize;
+}
+
+
+static unsigned long long selectDecoder( FILE* finput, FILE* foutput)
+{
+ unsigned int magicNumber, size;
+ int errorNb;
+ size_t nbReadBytes;
+
+ // Check Archive Header
+ nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);
+ if (nbReadBytes==0) return 0; // EOF
+ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");
+ magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format
+ if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers
+
+ switch(magicNumber)
+ {
+ case LZ4S_MAGICNUMBER:
+ return DEFAULT_DECOMPRESSOR(finput, foutput);
+ case LEGACY_MAGICNUMBER:
+ DISPLAYLEVEL(4, "Detected : Legacy format \n");
+ return decodeLegacyStream(finput, foutput);
+ case LZ4S_SKIPPABLE0:
+ DISPLAYLEVEL(4, "Skipping detected skippable area \n");
+ nbReadBytes = fread(&size, 1, 4, finput);
+ if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
+ size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format
+ errorNb = fseek(finput, size, SEEK_CUR);
+ if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
+ return selectDecoder(finput, foutput);
+ EXTENDED_FORMAT;
+ default:
+ if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream
+ DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
+ return 0;
+ }
+}
+
+
+int LZ4IO_decompressFilename(char* input_filename, char* output_filename)
+{
+ unsigned long long filesize = 0, decodedSize=0;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+
+
+ // Init
+ start = clock();
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+
+ // Loop over multiple streams
+ do
+ {
+ decodedSize = selectDecoder(finput, foutput);
+ filesize += decodedSize;
+ } while (decodedSize);
+
+ // Final Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close
+ fclose(finput);
+ fclose(foutput);
+
+ // Error status = OK
+ return 0;
+}
+
diff --git a/programs/lz4io.h b/programs/lz4io.h
new file mode 100644
index 0000000..cb35ec0
--- /dev/null
+++ b/programs/lz4io.h
@@ -0,0 +1,77 @@
+/*
+ LZ4io.h - LZ4 File/Stream Interface
+ Copyright (C) Yann Collet 2011-2013
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+/*
+ Note : this is stand-alone program.
+ It is not part of LZ4 compression library, it is a user code of the LZ4 library.
+ - The license of LZ4 library is BSD.
+ - The license of xxHash library is BSD.
+ - The license of this source file is GPLv2.
+*/
+
+
+/* ************************************************** */
+/* Special input/output values */
+/* ************************************************** */
+#define NULL_OUTPUT "null"
+static char stdinmark[] = "stdin";
+static char stdoutmark[] = "stdout";
+#ifdef _WIN32
+static char nulmark[] = "nul";
+#else
+static char nulmark[] = "/dev/null";
+#endif
+
+
+/* ************************************************** */
+/* ****************** Functions ********************* */
+/* ************************************************** */
+
+int LZ4IO_compressFilename (char* input_filename, char* output_filename, int compressionlevel);
+int LZ4IO_decompressFilename(char* input_filename, char* output_filename);
+
+
+/* ************************************************** */
+/* ****************** Parameters ******************** */
+/* ************************************************** */
+
+/* Default setting : overwrite = 1;
+ return : overwrite mode (0/1) */
+int LZ4IO_setOverwrite(int yes);
+
+/* blockSizeID : valid values : 4-5-6-7
+ return : -1 if error, blockSize if OK */
+int LZ4IO_setBlockSizeID(int blockSizeID);
+
+/* Default setting : independent blocks */
+typedef enum { chainedBlocks, independentBlocks } blockMode_t;
+int LZ4IO_setBlockMode(blockMode_t blockMode);
+
+/* Default setting : no checksum */
+int LZ4IO_setBlockChecksumMode(int xxhash);
+
+/* Default setting : checksum enabled */
+int LZ4IO_setStreamChecksumMode(int xxhash);
+
+/* Default setting : 0 (no notification) */
+int LZ4IO_setNotificationLevel(int level);