diff options
-rw-r--r-- | Makefile | 3 | ||||
-rw-r--r-- | NEWS | 6 | ||||
-rw-r--r-- | cmake_unofficial/CMakeLists.txt | 2 | ||||
-rw-r--r-- | lz4.c | 374 | ||||
-rw-r--r-- | lz4.h | 500 | ||||
-rw-r--r-- | lz4hc.c | 306 | ||||
-rw-r--r-- | lz4hc.h | 329 | ||||
-rw-r--r-- | programs/Makefile | 8 | ||||
-rw-r--r-- | programs/bench.c | 12 | ||||
-rw-r--r-- | programs/fuzzer.c | 5 | ||||
-rw-r--r-- | programs/lz4cli.c | 773 | ||||
-rw-r--r-- | programs/lz4io.c | 948 | ||||
-rw-r--r-- | programs/lz4io.h | 77 |
13 files changed, 1855 insertions, 1488 deletions
@@ -30,7 +30,7 @@ # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ################################################################ -RELEASE=r112 +RELEASE=r113 DESTDIR= PREFIX=/usr CC=gcc @@ -53,6 +53,7 @@ TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ lz4_format_description.txt Makefile NEWS LICENSE \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \ + $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ $(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \ $(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING @@ -1,3 +1,9 @@ +r113: +Large decompression speed improvement for GCC 32-bits. Thanks to Valery Croizier ! +LZ4HC : Compression Level is now a programmable parameter (CLI from 4 to 9) +Separated IO routines from command line (lz4io.c) +Version number into lz4.h (suggested by Francesc Alted) + r112: quickfix diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt index 356f18a..480654e 100644 --- a/cmake_unofficial/CMakeLists.txt +++ b/cmake_unofficial/CMakeLists.txt @@ -27,7 +27,7 @@ endif() set(LZ4_DIR ../) set(PRG_DIR ../programs/) set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h) -set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c) +set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) if(BUILD_TOOLS AND NOT BUILD_LIBS) set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2013, Yann Collet. + Copyright (C) 2011-2014, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -31,37 +31,43 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ -//************************************** -// Tuning parameters -//************************************** -// MEMORY_USAGE : -// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -// Increasing memory usage improves compression ratio -// Reduced memory usage can improve speed, due to cache effect -// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache +/************************************** + Tuning parameters +**************************************/ +/* + * MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ #define MEMORY_USAGE 14 -// HEAPMODE : -// Select how default compression functions will allocate memory for their hash table, -// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). + */ #define HEAPMODE 0 -//************************************** -// CPU Feature Detection -//************************************** -// 32 or 64 bits ? +/************************************** + CPU Feature Detection +**************************************/ +/* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ # define LZ4_ARCH64 1 #else # define LZ4_ARCH64 0 #endif -// Little Endian or Big Endian ? -// Overwrite the #define below if you know your architecture endianess +/* + * Little Endian or Big Endian ? + * Overwrite the #define below if you know your architecture endianess + */ #if defined (__GLIBC__) # include <endian.h> # if (__BYTE_ORDER == __BIG_ENDIAN) @@ -75,48 +81,53 @@ || defined(_MIPSEB) || defined(__s390__) # define LZ4_BIG_ENDIAN 1 #else -// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ #endif -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +/* + * Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance + */ #if defined(__ARM_FEATURE_UNALIGNED) # define LZ4_FORCE_UNALIGNED_ACCESS 1 #endif -// Define this parameter if your target system or compiler does not support hardware bit count -#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +/* Define this parameter if your target system or compiler does not support hardware bit count */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ # define LZ4_FORCE_SW_BITCOUNT #endif -// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : -// This option may provide a small boost to performance for some big endian cpu, although probably modest. -// You may set this option to 1 if data will remain within closed environment. -// This option is useless on Little_Endian CPU (such as x86) -//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 +/* + * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : + * This option may provide a small boost to performance for some big endian cpu, although probably modest. + * You may set this option to 1 if data will remain within closed environment. + * This option is useless on Little_Endian CPU (such as x86) + */ + +/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */ -//************************************** -// Compiler Options -//************************************** -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99 +/************************************** + Compiler Options +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ /* "restrict" is a known keyword */ #else -# define restrict // Disable restrict +# define restrict /* Disable restrict */ #endif -#ifdef _MSC_VER // Visual Studio +#ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline -# include <intrin.h> // For Visual 2005 -# if LZ4_ARCH64 // 64-bits -# pragma intrinsic(_BitScanForward64) // For Visual 2005 -# pragma intrinsic(_BitScanReverse64) // For Visual 2005 -# else // 32-bits -# pragma intrinsic(_BitScanForward) // For Visual 2005 -# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# include <intrin.h> /* For Visual 2005 */ +# if LZ4_ARCH64 /* 64-bits */ +# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ +# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ +# else /* 32-bits */ +# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ +# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ # endif -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #else # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) @@ -125,7 +136,7 @@ # endif #endif -#ifdef _MSC_VER +#ifdef _MSC_VER /* Visual Studio */ # define lz4_bswap16(x) _byteswap_ushort(x) #else # define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) @@ -143,26 +154,26 @@ #define unlikely(expr) expect((expr) != 0, 0) -//************************************** -// Memory routines -//************************************** -#include <stdlib.h> // malloc, calloc, free +/************************************** + Memory routines +**************************************/ +#include <stdlib.h> /* malloc, calloc, free */ #define ALLOCATOR(n,s) calloc(n,s) #define FREEMEM free -#include <string.h> // memset, memcpy +#include <string.h> /* memset, memcpy */ #define MEM_INIT memset -//************************************** -// Includes -//************************************** +/************************************** + Includes +**************************************/ #include "lz4.h" -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************** + Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -210,9 +221,9 @@ typedef struct {size_t v;} _PACKED size_t_S; #define AARCH(x) (((size_t_S *)(x))->v) -//************************************** -// Constants -//************************************** +/************************************** + Constants +**************************************/ #define LZ4_HASHLOG (MEMORY_USAGE-2) #define HASHTABLESIZE (1 << MEMORY_USAGE) #define HASHNBCELLS4 (1 << LZ4_HASHLOG) @@ -222,10 +233,14 @@ typedef struct {size_t v;} _PACKED size_t_S; #define COPYLENGTH 8 #define LASTLITERALS 5 #define MFLIMIT (COPYLENGTH+MINMATCH) -const int LZ4_minLength = (MFLIMIT+1); +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) -#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1)) -#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data +#define LZ4_64KLIMIT ((64 KB) + (MFLIMIT-1)) +#define SKIPSTRENGTH 6 /* Increasing this value will make the compression run slower on incompressible data */ #define MAXD_LOG 16 #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) @@ -235,15 +250,10 @@ const int LZ4_minLength = (MFLIMIT+1); #define RUN_BITS (8-ML_BITS) #define RUN_MASK ((1U<<RUN_BITS)-1) -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - - -//************************************** -// Structures and local types -//************************************** +/************************************** + Structures and local types +**************************************/ typedef struct { U32 hashTable[HASHNBCELLS4]; const BYTE* bufferStart; @@ -260,40 +270,36 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; -//************************************** -// Architecture-specific macros -//************************************** +/************************************** + Architecture-specific macros +**************************************/ #define STEPSIZE sizeof(size_t) #define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; } #define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); } -#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); } - -#if LZ4_ARCH64 // 64-bit -# define HTYPE U32 -# define INITBASE(base) const BYTE* const base = ip -#else // 32-bit -# define HTYPE const BYTE* -# define INITBASE(base) const int base = 0 -#endif #if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE)) # define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } # define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; } -#else // Little Endian +#else /* Little Endian */ # define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); } # define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } #endif -//************************************** -// Macros -//************************************** -#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e; +/************************************** + Macros +**************************************/ +#if LZ4_ARCH64 || !defined(__GNUC__) +# define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } /* at the end, d>=e; */ +#else +# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); } +#endif +#define LZ4_SECURECOPY(d,s,e) { if (d<e) LZ4_WILDCOPY(d,s,e); } -//**************************** -// Private functions -//**************************** +/**************************** + Private local functions +****************************/ #if LZ4_ARCH64 FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) @@ -360,9 +366,9 @@ FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) #endif -//**************************** -// Compression functions -//**************************** +/**************************** + Compression functions +****************************/ FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType) { if (tableType == byU16) @@ -393,7 +399,7 @@ FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType { if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) @@ -429,18 +435,18 @@ FORCE_INLINE int LZ4_compress_generic( const int skipStrength = SKIPSTRENGTH; U32 forwardH; - // Init conditions - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative) - if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block - if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit - if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit) - if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals) + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; /* must continue from end of previous block */ + if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; /* do it now, due to potential early exit */ + if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - // First Byte + /* First Byte */ LZ4_putPosition(ip, ctx, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); - // Main Loop + /* Main Loop */ for ( ; ; ) { int findMatchAttempts = (1U << skipStrength) + 3; @@ -448,14 +454,14 @@ FORCE_INLINE int LZ4_compress_generic( const BYTE* ref; BYTE* token; - // Find a match + /* Find a match */ do { U32 h = forwardH; int step = findMatchAttempts++ >> skipStrength; ip = forwardIp; forwardIp = ip + step; - if unlikely(forwardIp > mflimit) { goto _last_literals; } + if (unlikely(forwardIp > mflimit)) { goto _last_literals; } forwardH = LZ4_hashPosition(forwardIp, tableType); ref = LZ4_getPositionOnHash(h, ctx, tableType, base); @@ -463,13 +469,13 @@ FORCE_INLINE int LZ4_compress_generic( } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip))); - // Catch up - while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; } + /* Catch up */ + while ((ip>anchor) && (ref > lowLimit) && (unlikely(ip[-1]==ref[-1]))) { ip--; ref--; } - // Encode Literal length + /* Encode Literal length */ length = (int)(ip - anchor); token = op++; - if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit + if ((limitedOutput) && (unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend))) return 0; /* Check output limit */ if (length>=(int)RUN_MASK) { int len = length-RUN_MASK; @@ -479,17 +485,17 @@ FORCE_INLINE int LZ4_compress_generic( } else *token = (BYTE)(length<<ML_BITS); - // Copy Literals + /* Copy Literals */ { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; } _next_match: - // Encode Offset + /* Encode Offset */ LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref)); - // Start Counting - ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified + /* Start Counting */ + ip+=MINMATCH; ref+=MINMATCH; /* MinMatch already verified */ anchor = ip; - while likely(ip<matchlimit-(STEPSIZE-1)) + while (likely(ip<matchlimit-(STEPSIZE-1))) { size_t diff = AARCH(ref) ^ AARCH(ip); if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; } @@ -501,9 +507,9 @@ _next_match: if ((ip<matchlimit) && (*ref == *ip)) ip++; _endCount: - // Encode MatchLength + /* Encode MatchLength */ length = (int)(ip - anchor); - if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit + if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend))) return 0; /* Check output limit */ if (length>=(int)ML_MASK) { *token += ML_MASK; @@ -514,34 +520,34 @@ _endCount: } else *token += (BYTE)(length); - // Test end of chunk + /* Test end of chunk */ if (ip > mflimit) { anchor = ip; break; } - // Fill table + /* Fill table */ LZ4_putPosition(ip-2, ctx, tableType, base); - // Test next position + /* Test next position */ ref = LZ4_getPosition(ip, ctx, tableType, base); LZ4_putPosition(ip, ctx, tableType, base); if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } - // Prepare next loop + /* Prepare next loop */ anchor = ip++; forwardH = LZ4_hashPosition(ip, tableType); } _last_literals: - // Encode Last Literals + /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit + if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<<ML_BITS); memcpy(op, anchor, iend - anchor); op += iend-anchor; } - // End + /* End */ return (int) (((char*)op)-dest); } @@ -549,9 +555,9 @@ _last_literals: int LZ4_compress(const char* source, char* dest, int inputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries + void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */ #else - U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries + U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ #endif int result; @@ -569,9 +575,9 @@ int LZ4_compress(const char* source, char* dest, int inputSize) int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries + void* ctx = ALLOCATOR(HASHNBCELLS4, 4); /* Aligned on 4-bytes boundaries */ #else - U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries + U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ #endif int result; @@ -587,16 +593,16 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in } -//***************************** -// Using an external allocation -//***************************** +/***************************** + Using external allocation +*****************************/ int LZ4_sizeofState() { return 1 << MEMORY_USAGE; } int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) { - if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary + if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ MEM_INIT(state, 0, LZ4_sizeofState()); if (inputSize < (int)LZ4_64KLIMIT) @@ -608,7 +614,7 @@ int LZ4_compress_withState (void* state, const char* source, char* dest, int inp int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) { - if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary + if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ MEM_INIT(state, 0, LZ4_sizeofState()); if (inputSize < (int)LZ4_64KLIMIT) @@ -618,9 +624,9 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* } -//**************************** -// Stream functions -//**************************** +/**************************** + Stream functions +****************************/ int LZ4_sizeofStreamState() { @@ -637,7 +643,7 @@ FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base) int LZ4_resetStreamState(void* state, const char* inputBuffer) { - if ((((size_t)state) & 3) != 0) return 1; // Error : pointer is not aligned on 4-bytes boundary + if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer); return 0; } @@ -662,8 +668,8 @@ char* LZ4_slideInputBuffer (void* LZ4_Data) LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data; size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB); - if ( (lz4ds->base - delta > lz4ds->base) // underflow control - || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit + if ( (lz4ds->base - delta > lz4ds->base) /* underflow control */ + || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) /* close to 32-bits limit */ { size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base; int nH; @@ -700,27 +706,29 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha } -//**************************** -// Decompression functions -//**************************** +/**************************** + Decompression functions +****************************/ -// This generic decompression function cover all use cases. -// It shall be instanciated several times, using different sets of directives -// Note that it is essential this generic function is really inlined, -// in order to remove useless branches during compilation optimisation. +/* + * This generic decompression function cover all use cases. + * It shall be instanciated several times, using different sets of directives + * Note that it is essential this generic function is really inlined, + * in order to remove useless branches during compilation optimisation. + */ FORCE_INLINE int LZ4_decompress_generic( const char* source, char* dest, - int inputSize, // - int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer. + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - int endOnInput, // endOnOutputSize, endOnInputSize - int prefix64k, // noPrefix, withPrefix - int partialDecoding, // full, partial - int targetOutputSize // only used if partialDecoding==partial + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int prefix64k, /* noPrefix, withPrefix */ + int partialDecoding, /* full, partial */ + int targetOutputSize /* only used if partialDecoding==partial */ ) { - // Local Variables + /* Local Variables */ const BYTE* restrict ip = (const BYTE*) source; const BYTE* ref; const BYTE* const iend = ip + inputSize; @@ -730,23 +738,24 @@ FORCE_INLINE int LZ4_decompress_generic( BYTE* cpy; BYTE* oexit = op + targetOutputSize; - const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64 + /*const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; / static reduces speed for LZ4_decompress_safe() on GCC64 */ + const size_t dec32table[] = {4-0, 4-3, 4-2, 4-3, 4-0, 4-0, 4-0, 4-0}; /* static reduces speed for LZ4_decompress_safe() on GCC64 */ static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; - // Special cases - if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything - if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer - if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1); + /* Special cases */ + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - // Main Loop + /* Main Loop */ while (1) { unsigned token; size_t length; - // get runlength + /* get runlength */ token = *ip++; if ((length=(token>>ML_BITS)) == RUN_MASK) { @@ -758,36 +767,36 @@ FORCE_INLINE int LZ4_decompress_generic( } } - // copy literals + /* copy literals */ cpy = op+length; if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-COPYLENGTH))) { if (partialDecoding) { - if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer - if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ } memcpy(op, ip, length); ip += length; op += length; - break; // Necessarily EOF, due to parsing restrictions + break; /* Necessarily EOF, due to parsing restrictions */ } LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; - // get offset + /* get offset */ LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; - if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer + if ((prefix64k==noPrefix) && (unlikely(ref < (BYTE* const)dest))) goto _output_error; /* Error : offset outside destination buffer */ - // get matchlength + /* get matchlength */ if ((length=(token&ML_MASK)) == ML_MASK) { - while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token + while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) /* Ensure enough bytes remain for LASTLITERALS + token */ { unsigned s = *ip++; length += s; @@ -796,39 +805,42 @@ FORCE_INLINE int LZ4_decompress_generic( } } - // copy repeated sequence - if unlikely((op-ref)<(int)STEPSIZE) + /* copy repeated sequence */ + if (unlikely((op-ref)<(int)STEPSIZE)) { const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref]; op[0] = ref[0]; op[1] = ref[1]; op[2] = ref[2]; op[3] = ref[3]; - op += 4, ref += 4; ref -= dec32table[op-ref]; + /*op += 4, ref += 4; ref -= dec32table[op-ref]; A32(op) = A32(ref); - op += STEPSIZE-4; ref -= dec64; + op += STEPSIZE-4; ref -= dec64;*/ + ref += dec32table[op-ref]; + A32(op+4) = A32(ref); + op += STEPSIZE; ref -= dec64; } else { LZ4_COPYSTEP(op,ref); } cpy = op + length - (STEPSIZE-4); - if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4)) + if (unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4))) { - if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */ LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH)); while(op<cpy) *op++=*ref++; op=cpy; continue; } LZ4_WILDCOPY(op, ref, cpy); - op=cpy; // correction + op=cpy; /* correction */ } - // end of decoding + /* end of decoding */ if (endOnInput) - return (int) (((char*)op)-dest); // Nb of output bytes decoded + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ else - return (int) (((char*)ip)-source); // Nb of input bytes read + return (int) (((char*)ip)-source); /* Nb of input bytes read */ - // Overflow error detected + /* Overflow error detected */ _output_error: return (int) (-(((char*)ip)-source))-1; } @@ -856,7 +868,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int output int LZ4_decompress_fast(const char* source, char* dest, int outputSize) { -#ifdef _MSC_VER // This version is faster with Visual +#ifdef _MSC_VER /* This version is faster with Visual */ return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0); #else return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0); @@ -1,249 +1,251 @@ -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2013, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -//************************************** -// Compiler Options -//************************************** -#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio -# define inline __inline // Visual C is not C99, but supports some kind of inline -#endif - - -//**************************** -// Simple Functions -//**************************** - -int LZ4_compress (const char* source, char* dest, int inputSize); -int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize); - -/* -LZ4_compress() : - Compresses 'inputSize' bytes from 'source' into 'dest'. - Destination buffer must be already allocated, - and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() - inputSize : Max supported value is LZ4_MAX_INPUT_VALUE - return : the number of bytes written in buffer dest - or 0 if the compression fails - -LZ4_decompress_safe() : - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets -*/ - - -//**************************** -// Advanced Functions -//**************************** -#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes -#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } - -/* -LZ4_compressBound() : - Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) - primarily useful for memory allocation of output buffer. - inline function is recommended for the general case, - macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation). - - isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ - - -int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); - -/* -LZ4_compress_limitedOutput() : - Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This function never writes outside of provided output buffer. - - inputSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes written in buffer 'dest' - or 0 if the compression fails -*/ - - -int LZ4_decompress_fast (const char* source, char* dest, int outputSize); - -/* -LZ4_decompress_fast() : - outputSize : is the original (uncompressed) size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is malformed, the function will stop decoding and return a negative result. - note : This function is a bit faster than LZ4_decompress_safe() - This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet. - Use this function preferably into a trusted environment (data to decode comes from a trusted source). - Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes. -*/ - -int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize); - -/* -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'inputSize' at position 'source' - into output buffer 'dest' of size 'maxOutputSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ - - -//***************************** -// Using an external allocation -//***************************** -int LZ4_sizeofState(); -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* -These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. -To know how much memory must be allocated for the compression tables, use : -int LZ4_sizeofState(); - -Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0). - -The allocated memory can be provided to the compressions functions using 'void* state' parameter. -LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions. -They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). -*/ - - -//**************************** -// Streaming Functions -//**************************** - -void* LZ4_create (const char* inputBuffer); -int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize); -char* LZ4_slideInputBuffer (void* LZ4_Data); -int LZ4_free (void* LZ4_Data); - -/* -These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. -In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function : - -void* LZ4_create (const char* inputBuffer); -The result of the function is the (void*) pointer on the LZ4 Data Structure. -This pointer will be needed in all other functions. -If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. -The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. -To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(). -Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(), -but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one. -If next block does not begin immediately after the previous one, the compression will fail (return 0). - -When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : -char* LZ4_slideInputBuffer(void* LZ4_Data); -must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. -Note that, for this function to work properly, minimum size of an input buffer must be 192KB. -==> The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure. -*/ - -int LZ4_sizeofStreamState(); -int LZ4_resetStreamState(void* state, const char* inputBuffer); - -/* -These functions achieve the same result as : -void* LZ4_create (const char* inputBuffer); - -They are provided here to allow the user program to allocate memory using its own routines. - -To know how much space must be allocated, use LZ4_sizeofStreamState(); -Note also that space must be 4-bytes aligned. - -Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned on 4-bytes boundaries, and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamState() must be 0 is OK. -Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries). -*/ - - -int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize); -int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize); - -/* -*_withPrefix64k() : - These decoding functions work the same as their "normal name" versions, - but can use up to 64KB of data in front of 'char* dest'. - These functions are necessary to decode inter-dependant blocks. -*/ - - -//**************************** -// Obsolete Functions -//**************************** - -static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - -/* -These functions are deprecated and should no longer be used. -They are provided here for compatibility with existing user programs. -*/ - - - -#if defined (__cplusplus) -} -#endif +/*
+ LZ4 - Fast LZ compression algorithm
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/************************************** + Version +**************************************/ +#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ +#define LZ4_VERSION_MINOR 1 /* for minor interface/format changes */ +#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +
+
+/**************************************
+ Compiler Options
+**************************************/
+#if (defined(__GNUC__) && defined(__STRICT_ANSI__)) || (defined(_MSC_VER) && !defined(__cplusplus)) /* Visual Studio */
+# define inline __inline /* Visual C is not C99, but supports some kind of inline */
+#endif
+
+
+/**************************************
+ Simple Functions
+**************************************/
+
+int LZ4_compress (const char* source, char* dest, int inputSize);
+int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress() :
+ Compresses 'inputSize' bytes from 'source' into 'dest'.
+ Destination buffer must be already allocated,
+ and must be sized to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound()
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ return : the number of bytes written in buffer dest
+ or 0 if the compression fails
+
+LZ4_decompress_safe() :
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
+*/
+
+
+/**************************************
+ Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
+
+/*
+LZ4_compressBound() :
+ Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
+ primarily useful for memory allocation of output buffer.
+ inline function is recommended for the general case,
+ macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
+
+ isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
+ return : maximum output size in a "worst case" scenario
+ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+
+
+int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes written in buffer 'dest'
+ or 0 if the compression fails
+*/
+
+
+int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
+
+/*
+LZ4_decompress_fast() :
+ outputSize : is the original (uncompressed) size
+ return : the number of bytes read from the source buffer (in other words, the compressed size)
+ If the source stream is malformed, the function will stop decoding and return a negative result.
+ note : This function is a bit faster than LZ4_decompress_safe()
+ This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
+ Use this function preferably into a trusted environment (data to decode comes from a trusted source).
+ Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
+*/
+
+int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
+
+/*
+LZ4_decompress_safe_partial() :
+ This function decompress a compressed block of size 'inputSize' at position 'source'
+ into output buffer 'dest' of size 'maxOutputSize'.
+ The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+ reducing decompression time.
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+ Always control how many bytes were decoded.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+
+
+int LZ4_sizeofState();
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofState();
+
+Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+ Streaming Functions
+**************************************/
+void* LZ4_create (const char* inputBuffer);
+int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBuffer (void* LZ4_Data);
+int LZ4_free (void* LZ4_Data);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
+
+void* LZ4_create (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4 Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
+Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
+but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBuffer(void* LZ4_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
+*/
+
+int LZ4_sizeofStreamState();
+int LZ4_resetStreamState(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_create (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamState();
+Note also that space must be 4-bytes aligned.
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned on 4-bytes boundaries, and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamState() must be 0 is OK.
+Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
+*/
+
+
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
+
+/*
+*_withPrefix64k() :
+ These decoding functions work the same as their "normal name" versions,
+ but can use up to 64KB of data in front of 'char* dest'.
+ These functions are necessary to decode inter-dependant blocks.
+*/
+
+
+/**************************************
+ Obsolete Functions
+**************************************/
+/*
+These functions are deprecated and should no longer be used.
+They are provided here for compatibility with existing user programs.
+*/
+static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
+static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+
+
+
+#if defined (__cplusplus)
+}
+#endif
@@ -1,6 +1,6 @@ /* LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2013, Yann Collet. + Copyright (C) 2011-2014, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -31,31 +31,41 @@ - LZ4 source repository : http://code.google.com/p/lz4/ */ -//************************************** -// Memory routines -//************************************** -#include <stdlib.h> // calloc, free + + +/************************************** + Tuning Parameter +**************************************/ +#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 + + +/************************************** + Memory routines +**************************************/ +#include <stdlib.h> /* calloc, free */ #define ALLOCATOR(s) calloc(1,s) #define FREEMEM free -#include <string.h> // memset, memcpy +#include <string.h> /* memset, memcpy */ #define MEM_INIT memset -//************************************** -// CPU Feature Detection -//************************************** -// 32 or 64 bits ? +/************************************** + CPU Feature Detection +**************************************/ +/* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode + || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ # define LZ4_ARCH64 1 #else # define LZ4_ARCH64 0 #endif -// Little Endian or Big Endian ? -// Overwrite the #define below if you know your architecture endianess +/* + * Little Endian or Big Endian ? + * Overwrite the #define below if you know your architecture endianess + */ #if defined (__GLIBC__) # include <endian.h> # if (__BYTE_ORDER == __BIG_ENDIAN) @@ -69,43 +79,45 @@ || defined(_MIPSEB) || defined(__s390__) # define LZ4_BIG_ENDIAN 1 #else -// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ #endif -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +/* + * Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance + */ #if defined(__ARM_FEATURE_UNALIGNED) # define LZ4_FORCE_UNALIGNED_ACCESS 1 #endif -// Define this parameter if your target system or compiler does not support hardware bit count -#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +/* Define this parameter if your target system or compiler does not support hardware bit count */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ # define LZ4_FORCE_SW_BITCOUNT #endif -//************************************** -// Compiler Options -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 - /* "restrict" is a known keyword */ +/************************************** + Compiler Options +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +/* "restrict" is a known keyword */ #else -# define restrict // Disable restrict +# define restrict /* Disable restrict */ #endif -#ifdef _MSC_VER // Visual Studio +#ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline -# include <intrin.h> // For Visual 2005 -# if LZ4_ARCH64 // 64-bits -# pragma intrinsic(_BitScanForward64) // For Visual 2005 -# pragma intrinsic(_BitScanReverse64) // For Visual 2005 -# else // 32-bits -# pragma intrinsic(_BitScanForward) // For Visual 2005 -# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# include <intrin.h> /* For Visual 2005 */ +# if LZ4_ARCH64 /* 64-bits */ +# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ +# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ +# else /* 32-bits */ +# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ +# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ # endif -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */ #else # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) @@ -114,24 +126,24 @@ # endif #endif -#ifdef _MSC_VER // Visual Studio +#ifdef _MSC_VER /* Visual Studio */ # define lz4_bswap16(x) _byteswap_ushort(x) #else # define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) #endif -//************************************** -// Includes -//************************************** +/************************************** + Includes +**************************************/ #include "lz4hc.h" #include "lz4.h" -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************** + Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -173,9 +185,9 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; #define A16(x) (((U16_S *)(x))->v) -//************************************** -// Constants -//************************************** +/************************************** + Constants +**************************************/ #define MINMATCH 4 #define DICTIONARY_LOGSIZE 16 @@ -187,8 +199,6 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; #define HASHTABLESIZE (1 << HASH_LOG) #define HASH_MASK (HASHTABLESIZE - 1) -#define MAX_NB_ATTEMPTS 256 - #define ML_BITS 4 #define ML_MASK (size_t)((1U<<ML_BITS)-1) #define RUN_BITS (8-ML_BITS) @@ -205,25 +215,21 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; #define GB *(1U<<30) -//************************************** -// Architecture-specific macros -//************************************** -#if LZ4_ARCH64 // 64-bit +/************************************** + Architecture-specific macros +**************************************/ +#if LZ4_ARCH64 /* 64-bit */ # define STEPSIZE 8 # define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8; # define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d) -# define UARCH U64 # define AARCH A64 # define HTYPE U32 # define INITBASE(b,s) const BYTE* const b = s -#else // 32-bit +#else /* 32-bit */ # define STEPSIZE 4 # define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4; # define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d); -# define UARCH U32 # define AARCH A32 -//# define HTYPE const BYTE* -//# define INITBASE(b,s) const int b = 0 # define HTYPE U32 # define INITBASE(b,s) const BYTE* const b = s #endif @@ -231,15 +237,15 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S; #if defined(LZ4_BIG_ENDIAN) # define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } # define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; } -#else // Little Endian +#else /* Little Endian */ # define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); } # define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; } #endif -//************************************************************ -// Local Types -//************************************************************ +/************************************** + Local Types +**************************************/ typedef struct { const BYTE* inputBuffer; @@ -251,9 +257,9 @@ typedef struct } LZ4HC_Data_Structure; -//************************************** -// Macros -//************************************** +/************************************** + Macros +**************************************/ #define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e); #define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; } #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) @@ -263,9 +269,9 @@ typedef struct #define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) -//************************************** -// Private functions -//************************************** +/************************************** + Private functions +**************************************/ #if LZ4_ARCH64 FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) @@ -349,7 +355,7 @@ FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base) int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) { - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; // Error : pointer is not aligned for pointer (32 or 64 bits) + if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); return 0; } @@ -370,7 +376,7 @@ int LZ4_freeHC (void* LZ4HC_Data) } -// Update chains up to ip (excluded) +/* Update chains up to ip (excluded) */ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) { U16* chainTable = hc4->chainTable; @@ -393,12 +399,12 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB; - distance = (distance >> 16) << 16; // Must be a multiple of 64 KB + distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ LZ4HC_Insert(hc4, hc4->end - MINMATCH); memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); hc4->nextToUpdate -= distance; hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow + if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ { int i; hc4->base += 1 GB; @@ -415,7 +421,7 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY while (p1t<matchlimit-(STEPSIZE-1)) { - UARCH diff = AARCH(p2) ^ AARCH(p1t); + size_t diff = AARCH(p2) ^ AARCH(p1t); if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; } p1t += LZ4_NbCommonBytes(diff); return (p1t - p1); @@ -427,26 +433,26 @@ FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BY } -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos) +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; HTYPE* const HashTable = hc4->hashTable; const BYTE* ref; INITBASE(base,hc4->base); - int nbAttempts=MAX_NB_ATTEMPTS; + int nbAttempts=maxNbAttempts; size_t repl=0, ml=0; - U16 delta=0; // useless assignment, to remove an uninitialization warning + U16 delta=0; /* useless assignment, to remove an uninitialization warning */ - // HC4 match finder + /* HC4 match finder */ LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); #define REPEAT_OPTIMIZATION #ifdef REPEAT_OPTIMIZATION - // Detect repetitive sequences of length <= 4 - if ((U32)(ip-ref) <= 4) // potential repetition + /* Detect repetitive sequences of length <= 4 */ + if ((U32)(ip-ref) <= 4) /* potential repetition */ { - if (A32(ref) == A32(ip)) // confirmed + if (A32(ref) == A32(ip)) /* confirmed */ { delta = (U16)(ip-ref); repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; @@ -469,7 +475,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const } #ifdef REPEAT_OPTIMIZATION - // Complete table + /* Complete table */ if (repl) { const BYTE* ptr = ip; @@ -478,13 +484,13 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const end = ip + repl - (MINMATCH-1); while(ptr < end-delta) { - DELTANEXT(ptr) = delta; // Pre-Load + DELTANEXT(ptr) = delta; /* Pre-Load */ ptr++; } do { DELTANEXT(ptr) = delta; - HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain + HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */ ptr++; } while(ptr < end); hc4->nextToUpdate = end; @@ -495,16 +501,16 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const } -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos) +FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; HTYPE* const HashTable = hc4->hashTable; INITBASE(base,hc4->base); const BYTE* ref; - int nbAttempts = MAX_NB_ATTEMPTS; + int nbAttempts = maxNbAttempts; int delta = (int)(ip-startLimit); - // First Match + /* First Match */ LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); @@ -521,7 +527,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const while (ipt<matchlimit-(STEPSIZE-1)) { - UARCH diff = AARCH(reft) ^ AARCH(ipt); + size_t diff = AARCH(reft) ^ AARCH(ipt); if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; } ipt += LZ4_NbCommonBytes(diff); goto _endCount; @@ -532,7 +538,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const _endCount: reft = ref; #else - // Easier for code maintenance, but unfortunately slower too + /* Easier for code maintenance, but unfortunately slower too */ const BYTE* startt = ip; const BYTE* reft = ref; const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit); @@ -568,26 +574,26 @@ FORCE_INLINE int LZ4HC_encodeSequence ( int length; BYTE* token; - // Encode Literal length + /* Encode Literal length */ length = (int)(*ip - *anchor); token = (*op)++; - if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit + if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */ if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } else *token = (BYTE)(length<<ML_BITS); - // Copy Literals + /* Copy Literals */ LZ4_BLINDCOPY(*anchor, *op, length); - // Encode Offset + /* Encode Offset */ LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref)); - // Encode MatchLength + /* Encode MatchLength */ length = (int)(matchLength-MINMATCH); - if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit + if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; /* Check output limit */ if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } else *token += (BYTE)(length); - // Prepare next loop + /* Prepare next loop */ *ip += matchLength; *anchor = *ip; @@ -595,12 +601,14 @@ FORCE_INLINE int LZ4HC_encodeSequence ( } +#define MAX_COMPRESSION_LEVEL 16 static int LZ4HC_compress_generic ( void* ctxvoid, const char* source, char* dest, int inputSize, int maxOutputSize, + int compressionLevel, limitedOutput_directive limit ) { @@ -614,6 +622,7 @@ static int LZ4HC_compress_generic ( BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; + const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<<LZ4HC_DEFAULT_COMPRESSIONLEVEL; int ml, ml2, ml3, ml0; const BYTE* ref=NULL; const BYTE* start2=NULL; @@ -624,29 +633,29 @@ static int LZ4HC_compress_generic ( const BYTE* ref0; - // Ensure blocks follow each other + /* Ensure blocks follow each other */ if (ip != ctx->end) return 0; ctx->end += inputSize; ip++; - // Main Loop + /* Main Loop */ while (ip < mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); if (!ml) { ip++; continue; } - // saved, in case we would skip too much + /* saved, in case we would skip too much */ start0 = ip; ref0 = ref; ml0 = ml; _Search2: if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); else ml2 = ml; - if (ml2 == ml) // No better match + if (ml2 == ml) /* No better match */ { if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; continue; @@ -654,7 +663,7 @@ _Search2: if (start0 < ip) { - if (start2 < ip + ml0) // empirical + if (start2 < ip + ml0) /* empirical */ { ip = start0; ref = ref0; @@ -662,8 +671,8 @@ _Search2: } } - // Here, start0==ip - if ((start2 - ip) < 3) // First Match too small : removed + /* Here, start0==ip */ + if ((start2 - ip) < 3) /* First Match too small : removed */ { ml = ml2; ip = start2; @@ -672,9 +681,11 @@ _Search2: } _Search3: - // Currently we have : - // ml2 > ml1, and - // ip1+3 <= ip2 (usually < ip1+ml1) + /* + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ if ((start2 - ip) < OPTIMAL_ML) { int correction; @@ -689,26 +700,26 @@ _Search3: ml2 -= correction; } } - // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); else ml3 = ml2; - if (ml3 == ml2) // No better match : 2 sequences to encode + if (ml3 == ml2) /* No better match : 2 sequences to encode */ { - // ip & ref are known; Now for ml + /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); - // Now, encode 2 sequences + /* Now, encode 2 sequences */ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; ip = start2; if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; continue; } - if (start3 < ip+ml+3) // Not enough space for match 2 : remove it + if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ { - if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 + if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ { if (start2 < ip+ml) { @@ -741,8 +752,10 @@ _Search3: goto _Search3; } - // OK, now we have 3 ascending matches; let's write at least the first one - // ip & ref are known; Now for ml + /* + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ if (start2 < ip+ml) { if ((start2 - ip) < (int)ML_MASK) @@ -777,80 +790,101 @@ _Search3: } - // Encode Last Literals + /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<<ML_BITS); memcpy(op, anchor, iend - anchor); op += iend-anchor; } - // End + /* End */ return (int) (((char*)op)-dest); } -int LZ4_compressHC(const char* source, char* dest, int inputSize) +int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel) { void* ctx = LZ4_createHC(source); int result; if (ctx==NULL) return 0; - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit); + result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit); LZ4_freeHC(ctx); return result; } -int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); } + +int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { void* ctx = LZ4_createHC(source); int result; if (ctx==NULL) return 0; - result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput); + result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); LZ4_freeHC(ctx); return result; } +int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0); +} -//***************************** -// Using an external allocation -//***************************** +/***************************** + Using external allocation +*****************************/ int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); } -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize) +int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits) + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, 0, noLimit); + return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit); } +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize) +{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); } -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) + +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits) + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } + -//**************************** -// Stream functions -//**************************** +/**************************** + Stream functions +****************************/ int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit); + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); +} + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); } +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); +} @@ -1,157 +1,172 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Header File - Copyright (C) 2011-2013, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ -#pragma once - - -#if defined (__cplusplus) -extern "C" { -#endif - - -int LZ4_compressHC (const char* source, char* dest, int inputSize); -/* -LZ4_compressHC : - return : the number of bytes in compressed buffer dest - or 0 if compression fails. - note : destination buffer must be already allocated. - To avoid any problem, size it to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") -*/ - -int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -/* -LZ4_compress_limitedOutput() : - Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This function never writes outside of provided output buffer. - - inputSize : Max supported value is 1 GB - maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) - return : the number of output bytes written in buffer 'dest' - or 0 if compression fails. -*/ - - -/* Note : -Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) -*/ - - -//***************************** -// Using an external allocation -//***************************** -int LZ4_sizeofStateHC(); -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* -These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. -To know how much memory must be allocated for the compression tables, use : -int LZ4_sizeofStateHC(); - -Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). - -The allocated memory can be provided to the compressions functions using 'void* state' parameter. -LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. -They just use the externally allocated memory area instead of allocating their own (on stack, or on heap). -*/ - - -//**************************** -// Streaming Functions -//**************************** - -void* LZ4_createHC (const char* inputBuffer); -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); -char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -int LZ4_freeHC (void* LZ4HC_Data); - -/* -These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. -In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : - -void* LZ4_createHC (const char* inputBuffer); -The result of the function is the (void*) pointer on the LZ4HC Data Structure. -This pointer will be needed in all other functions. -If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. -The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. -To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). -Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), -but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. -If next block does not begin immediately after the previous one, the compression will fail (return 0). - -When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : -char* LZ4_slideInputBufferHC(void* LZ4HC_Data); -must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. -Note that, for this function to work properly, minimum size of an input buffer must be 192KB. -==> The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. -*/ - -int LZ4_sizeofStreamStateHC(); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); - -/* -These functions achieve the same result as : -void* LZ4_createHC (const char* inputBuffer); - -They are provided here to allow the user program to allocate memory using its own routines. - -To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); -Note also that space must be aligned for pointers (32 or 64 bits). - -Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned for pointers (32 or 64 bits), and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamStateHC() must be 0 is OK. -Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). -*/ - - -#if defined (__cplusplus) -} -#endif +/*
+ LZ4 HC - High Compression Mode of LZ4
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+int LZ4_compressHC (const char* source, char* dest, int inputSize);
+/*
+LZ4_compressHC :
+ return : the number of bytes in compressed buffer dest
+ or 0 if compression fails.
+ note : destination buffer must be already allocated.
+ To avoid any problem, size it to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+*/
+
+int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is 1 GB
+ maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
+ return : the number of output bytes written in buffer 'dest'
+ or 0 if compression fails.
+*/
+
+
+int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+/*
+ Same functions as above, but with programmable 'compressionLevel'.
+ Recommended values are between 4 and 9, although any value between 0 and 16 will work.
+ 'compressionLevel'==0 means use default 'compressionLevel' value.
+ Values above 16 behave the same as 16.
+ Equivalent variants exist for all other compression functions below.
+*/
+
+/* Note :
+Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+*/
+
+
+/**************************************
+ Using an external allocation
+**************************************/
+int LZ4_sizeofStateHC();
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofStateHC();
+
+Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+ Streaming Functions
+**************************************/
+void* LZ4_createHC (const char* inputBuffer);
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+int LZ4_freeHC (void* LZ4HC_Data);
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
+
+void* LZ4_createHC (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4HC Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
+Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
+but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
+*/
+
+int LZ4_sizeofStreamStateHC();
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_createHC (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
+Note also that space must be aligned for pointers (32 or 64 bits).
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned for pointers (32 or 64 bits), and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamStateHC() must be 0 is OK.
+Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
+*/
+
+
+#if defined (__cplusplus)
+}
+#endif
diff --git a/programs/Makefile b/programs/Makefile index 5f86d3c..98ed185 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ################################################################ -RELEASE=r112 +RELEASE=r113 DESTDIR= PREFIX=/usr CC=gcc @@ -53,13 +53,13 @@ default: lz4 lz4c all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32 -lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c +lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c $(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c +lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) -lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c +lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4io.c lz4cli.c $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c diff --git a/programs/bench.c b/programs/bench.c index df3c44a..ea6a1ca 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -59,9 +59,10 @@ #endif #include "lz4.h" -#define COMPRESSOR0 LZ4_compress +#define COMPRESSOR0 LZ4_compress_local +static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); } #include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC +#define COMPRESSOR1 LZ4_compressHC2 #define DEFAULTCOMPRESSOR COMPRESSOR0 #include "xxhash.h" @@ -128,7 +129,7 @@ struct chunkParameters struct compressionParameters { - int (*compressionFunction)(const char*, char*, int); + int (*compressionFunction)(const char*, char*, int, int); int (*decompressionFunction)(const char*, char*, int); }; @@ -139,7 +140,6 @@ struct compressionParameters #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - //************************************** // Benchmark Parameters //************************************** @@ -254,7 +254,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) // Init - if (cLevel <3) cfunctionId = 0; else cfunctionId = 1; + if (cLevel <= 3) cfunctionId = 0; else cfunctionId = 1; switch (cfunctionId) { #ifdef COMPRESSOR0 @@ -377,7 +377,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { for (chunkNb=0; chunkNb<nbChunks; chunkNb++) - chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); + chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel); nbLoops++; } milliTime = BMK_GetMilliSpan(milliTime); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 4513ebe..7b06cbe 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -1,7 +1,6 @@ /* fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet - Andrew Mahone 2012-2013 - Code started by Andrew Mahone, modified by Yann Collet + Copyright (C) Yann Collet - Andrew Mahone 2012-2014 GPL v2 License This program is free software; you can redistribute it and/or modify @@ -26,7 +25,7 @@ //************************************** // Remove Visual warning messages //************************************** -#define _CRT_SECURE_NO_WARNINGS // fgets +#define _CRT_SECURE_NO_WARNINGS // fgets //************************************** diff --git a/programs/lz4cli.c b/programs/lz4cli.c index f69c8a0..26e0111 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -63,6 +63,7 @@ #include "lz4hc.h" #include "xxhash.h" #include "bench.h" +#include "lz4io.h" //**************************** @@ -108,7 +109,7 @@ //**************************** #define COMPRESSOR_NAME "LZ4 Compression CLI" #ifndef LZ4_VERSION -# define LZ4_VERSION "v1.1.0" +# define LZ4_VERSION "v1.1.3" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ @@ -118,25 +119,7 @@ #define MB *(1U<<20) #define GB *(1U<<30) -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define MAGICNUMBER_SIZE 4 -#define LZ4S_MAGICNUMBER 0x184D2204 -#define LZ4S_SKIPPABLE0 0x184D2A50 -#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 -#define LEGACY_MAGICNUMBER 0x184C2102 - -#define CACHELINE 64 -#define LEGACY_BLOCKSIZE (8 MB) -#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) -#define LZ4S_BLOCKSIZEID_DEFAULT 7 -#define LZ4S_CHECKSUM_SEED 0 -#define LZ4S_EOS 0 -#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) +#define LZ4_BLOCKSIZEID_DEFAULT 7 //************************************** @@ -156,28 +139,10 @@ static const int one = 1; //************************************** -// Special input/output -//************************************** -#define NULL_OUTPUT "null" -char stdinmark[] = "stdin"; -char stdoutmark[] = "stdout"; -#ifdef _WIN32 -char nulmark[] = "nul"; -#else -char nulmark[] = "/dev/null"; -#endif - - -//************************************** // Local Parameters //************************************** static char* programName; static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information -static int overwrite = 0; -static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; -static int blockChecksum = 0; -static int streamChecksum = 1; -static int blockIndependence = 1; //************************************** @@ -201,8 +166,9 @@ static int blockIndependence = 1; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR compress_file -#define DEFAULT_DECOMPRESSOR decodeLZ4S +#define DEFAULT_COMPRESSOR LZ4IO_compressFilename +#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename +int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel); //**************************** @@ -314,705 +280,6 @@ int badusage() } -static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } -static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } - - -int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) -{ - - if (!strcmp (input_filename, stdinmark)) - { - DISPLAYLEVEL(4,"Using stdin for input\n"); - *pfinput = stdin; - SET_BINARY_MODE(stdin); - } - else - { - *pfinput = fopen(input_filename, "rb"); - } - - if (!strcmp (output_filename, stdoutmark)) - { - DISPLAYLEVEL(4,"Using stdout for output\n"); - *pfoutput = stdout; - SET_BINARY_MODE(stdout); - } - else - { - // Check if destination file already exists - *pfoutput=0; - if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); - if (*pfoutput!=0) - { - fclose(*pfoutput); - if (!overwrite) - { - char ch; - DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); - DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible - ch = (char)getchar(); - if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); - } - } - *pfoutput = fopen( output_filename, "wb" ); - } - - if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); - if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); - - return 0; -} - - - -int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel) -{ - int (*compressionFunction)(const char*, char*, int); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = MAGICNUMBER_SIZE; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; - int displayLevel = (compressionlevel>0); - clock_t start, end; - size_t sizeCheck; - - - // Init - if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; - - // Allocate Memory - in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); - - // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); - sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); - - // Main Loop - while (1) - { - unsigned int outSize; - // Read Block - int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); - if( inSize<=0 ) break; - filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - - // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, inSize); - compressedfilesize += outSize+4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); - } - - // Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) -{ - void* (*initFunction) (const char*); - int (*compressionFunction)(void*, const char*, char*, int, int); - char* (*translateFunction) (void*); - int (*freeFunction) (void*); - void* ctx; - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - unsigned int checkbits; - char* in_buff, *in_start, *in_end; - char* out_buff; - FILE* finput; - FILE* foutput; - clock_t start, end; - unsigned int blockSize, inputBufferSize; - size_t sizeCheck, header_size; - void* streamChecksumState=NULL; - - - // Init - start = clock(); - if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - if (compressionlevel>=3) - { - initFunction = LZ4_createHC; - compressionFunction = LZ4_compressHC_limitedOutput_continue; - translateFunction = LZ4_slideInputBufferHC; - freeFunction = LZ4_freeHC; - } - else - { - initFunction = LZ4_create; - compressionFunction = LZ4_compress_limitedOutput_continue; - translateFunction = LZ4_slideInputBuffer; - freeFunction = LZ4_free; - } - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); - - // Allocate Memory - inputBufferSize = blockSize + 64 KB; - if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE; - in_buff = (char*)malloc(inputBufferSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); - in_start = in_buff; in_end = in_buff + inputBufferSize; - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - ctx = initFunction(in_buff); - - // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') - *(out_buff+4) |= (blockIndependence & _1BIT) << 5; - *(out_buff+4) |= (blockChecksum & _1BIT) << 4; - *(out_buff+4) |= (streamChecksum & _1BIT) << 2; - *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4); - checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(out_buff+6) = (unsigned char) checkbits; - header_size = 7; - sizeCheck = fwrite(out_buff, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; - - // Main Loop - while (1) - { - unsigned int outSize; - unsigned int inSize; - // Read Block - if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); - inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); - if( inSize==0 ) break; // No more input : end of compression - filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); - - // Compress Block - outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - - } - else // Copy Original - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_start, 1, inSize, foutput); - if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } - in_start += inSize; - } - - // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } - - // Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - freeFunction(ctx); - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -int compress_file(char* input_filename, char* output_filename, int compressionlevel) -{ - int (*compressionFunction)(const char*, char*, int, int); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - unsigned int checkbits; - char* in_buff; - char* out_buff; - char* headerBuffer; - FILE* finput; - FILE* foutput; - clock_t start, end; - int blockSize; - size_t sizeCheck, header_size, readSize; - void* streamChecksumState=NULL; - - // Branch out - if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel); - - // Init - start = clock(); - if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput; - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); - - // Allocate Memory - in_buff = (char*)malloc(blockSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); - if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - - // Write Archive Header - *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') - *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; - *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; - *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; - *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4); - checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(headerBuffer+6) = (unsigned char) checkbits; - header_size = 7; - - // Write header - sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; - - // read first block - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - - // Main Loop - while (readSize>0) - { - unsigned int outSize; - - filesize += readSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); - - // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - } - else // Copy Original Uncompressed - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_buff, 1, readSize, foutput); - if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } - - // Read next block - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - } - - // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } - - // Close & Free - free(in_buff); - free(out_buff); - free(headerBuffer); - fclose(finput); - fclose(foutput); - - // Final Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - return 0; -} - - -unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) -{ - unsigned long long filesize = 0; - char* in_buff; - char* out_buff; - unsigned int blockSize; - - - // Allocate Memory - in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - out_buff = (char*)malloc(LEGACY_BLOCKSIZE); - if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); - - // Main Loop - while (1) - { - int decodeSize; - size_t sizeCheck; - - // Block Size - sizeCheck = fread(&blockSize, 1, 4, finput); - if (sizeCheck==0) break; // Nothing to read : file read is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian - if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) - { // Cannot read next block : maybe new stream ? - fseek(finput, -4, SEEK_CUR); - break; - } - - // Read Block - sizeCheck = fread(in_buff, 1, blockSize, finput); - - // Decode Block - decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); - filesize += decodeSize; - - // Write Block - sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); - } - - // Free - free(in_buff); - free(out_buff); - - return filesize; -} - - -unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) -{ - unsigned long long filesize = 0; - char* in_buff; - char* out_buff, *out_start, *out_end; - unsigned char descriptor[LZ4S_MAXHEADERSIZE]; - size_t nbReadBytes; - int decodedBytes=0; - unsigned int maxBlockSize; - size_t sizeCheck; - int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; - void* streamChecksumState=NULL; - int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; - unsigned int prefix64k = 0; - - // Decode stream descriptor - nbReadBytes = fread(descriptor, 1, 3, finput); - if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); - { - int version = (descriptor[0] >> 6) & _2BITS; - int streamSize = (descriptor[0] >> 3) & _1BIT; - int reserved1 = (descriptor[0] >> 1) & _1BIT; - int dictionary = (descriptor[0] >> 0) & _1BIT; - - int reserved2 = (descriptor[1] >> 7) & _1BIT; - int blockSizeId = (descriptor[1] >> 4) & _3BITS; - int reserved3 = (descriptor[1] >> 0) & _4BITS; - int checkBits = (descriptor[2] >> 0) & _8BITS; - int checkBits_xxh32; - - blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; - blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; - streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; - - if (version != 1) EXM_THROW(62, "Wrong version number"); - if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); - if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); - if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); - if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); - if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); - // Checkbits verification - descriptor[1] &= 0xF0; - checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); - checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); - if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); - } - - if (!blockIndependenceFlag) - { - decompressionFunction = LZ4_decompress_safe_withPrefix64k; - prefix64k = 64 KB; - } - - // Allocate Memory - { - unsigned int outbuffSize = prefix64k+maxBlockSize; - in_buff = (char*)malloc(maxBlockSize); - if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; - out_buff = (char*)malloc(outbuffSize); - out_end = out_buff + outbuffSize; - out_start = out_buff + prefix64k; - if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); - } - if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - - // Main Loop - while (1) - { - unsigned int blockSize, uncompressedFlag; - - // Block Size - nbReadBytes = fread(&blockSize, 1, 4, finput); - if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); - if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian - uncompressedFlag = blockSize >> 31; - blockSize &= 0x7FFFFFFF; - if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); - - // Read Block - nbReadBytes = fread(in_buff, 1, blockSize, finput); - if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); - - // Check Block - if (blockChecksumFlag) - { - unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); - } - - if (uncompressedFlag) - { - // Write uncompressed Block - sizeCheck = fwrite(in_buff, 1, blockSize, foutput); - if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); - filesize += blockSize; - if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); - if (!blockIndependenceFlag) - { - if (blockSize >= prefix64k) - { - memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks - out_start = out_buff + prefix64k; - continue; - } - else - { - memcpy(out_start, in_buff, blockSize); - decodedBytes = blockSize; - } - } - } - else - { - // Decode Block - decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); - if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); - filesize += decodedBytes; - if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); - - // Write Block - sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); - if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); - } - - if (!blockIndependenceFlag) - { - out_start += decodedBytes; - if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) - { - memcpy(out_buff, out_start - prefix64k, prefix64k); - out_start = out_buff + prefix64k; - } - } - } - - // Stream Checksum - if (streamChecksumFlag) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); - } - - // Free - free(in_buff); - free(out_buff); - - return filesize; -} - - -unsigned long long selectDecoder( FILE* finput, FILE* foutput) -{ - unsigned int magicNumber, size; - int errorNb; - size_t nbReadBytes; - - // Check Archive Header - nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) return 0; // EOF - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); - magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format - if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers - - switch(magicNumber) - { - case LZ4S_MAGICNUMBER: - return DEFAULT_DECOMPRESSOR(finput, foutput); - case LEGACY_MAGICNUMBER: - DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return decodeLegacyStream(finput, foutput); - case LZ4S_SKIPPABLE0: - DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(&size, 1, 4, finput); - if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format - errorNb = fseek(finput, size, SEEK_CUR); - if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - return selectDecoder(finput, foutput); - EXTENDED_FORMAT; - default: - if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream - DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); - return 0; - } -} - - -int decodeFile(char* input_filename, char* output_filename) -{ - unsigned long long filesize = 0, decodedSize=0; - FILE* finput; - FILE* foutput; - clock_t start, end; - - - // Init - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - - // Loop over multiple streams - do - { - decodedSize = selectDecoder(finput, foutput); - filesize += decodedSize; - } while (decodedSize); - - // Final Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close - fclose(finput); - fclose(foutput); - - // Error status = OK - return 0; -} - - void waitEnter() { DISPLAY("Press enter to continue...\n"); @@ -1036,9 +303,12 @@ int main(int argc, char** argv) char* dynNameSpace=0; char nullOutput[] = NULL_OUTPUT; char extension[] = LZ4_EXTENSION; + int blockSize; // Init programName = argv[0]; + LZ4IO_setOverwrite(0); + blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); for(i=1; i<argc; i++) { @@ -1065,7 +335,7 @@ int main(int argc, char** argv) if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression) if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression) if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression) - if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission) + if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } // -y (answer 'yes' to overwrite permission) if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode) #endif // DISABLE_LZ4C_LEGACY_OPTIONS @@ -1089,7 +359,9 @@ int main(int argc, char** argv) case '6': case '7': case '8': - case '9': cLevel=*argument -'0'; break; + case '9': + case 'A': /* non documented (hidden) */ + cLevel=*argument -'0'; break; // Use Legacy format (for Linux kernel compression) case 'l': legacy_format=1; break; @@ -1104,7 +376,7 @@ int main(int argc, char** argv) case 't': decode=1; output_filename=nulmark; break; // Overwrite - case 'f': overwrite=1; break; + case 'f': LZ4IO_setOverwrite(1); break; // Verbose mode case 'v': displayLevel=4; break; @@ -1130,12 +402,12 @@ int main(int argc, char** argv) int B = argument[1] - '0'; int S = 1 << (8 + 2*B); BMK_SetBlocksize(S); - blockSizeId = B; + LZ4IO_setBlockSizeID(B); argument++; break; } - case 'D': blockIndependence = 0, argument++; break; - case 'X': blockChecksum = 1, argument ++; break; + case 'D': LZ4IO_setBlockMode(independentBlocks); argument++; break; + case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; default : exitBlockProperties=1; } if (exitBlockProperties) break; @@ -1143,7 +415,7 @@ int main(int argc, char** argv) break; // Modify Stream properties - case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); } + case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); } // Benchmark case 'b': bench=1; break; @@ -1183,7 +455,7 @@ int main(int argc, char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); - DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2))); + DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); // No input filename ==> use stdin if(!input_filename) { input_filename=stdinmark; } @@ -1235,15 +507,16 @@ int main(int argc, char** argv) if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - // Decompress input if selected - if (decode) decodeFile(input_filename, output_filename); + // IO Stream/File + LZ4IO_setNotificationLevel(displayLevel); + if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename); else // compression is default action { if (legacy_format) { DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); - legacy_compress_file(input_filename, output_filename, cLevel); + LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); } else { diff --git a/programs/lz4io.c b/programs/lz4io.c new file mode 100644 index 0000000..ae8b776 --- /dev/null +++ b/programs/lz4io.c @@ -0,0 +1,948 @@ +/* + LZ4io.c - LZ4 File/Stream Interface + Copyright (C) Yann Collet 2011-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* + Note : this is stand-alone program. + It is not part of LZ4 compression library, it is a user code of the LZ4 library. + - The license of LZ4 library is BSD. + - The license of xxHash library is BSD. + - The license of this source file is GPLv2. +*/ + +//************************************** +// Compiler Options +//************************************** +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# define _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_DEPRECATE // VS2005 +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix +#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix + + +//**************************** +// Includes +//**************************** +#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout +#include <stdlib.h> // malloc +#include <string.h> // strcmp, strlen +#include <time.h> // clock +#include "lz4io.h" +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" + + +//**************************** +// OS-specific Includes +//**************************** +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include <fcntl.h> // _O_BINARY +# include <io.h> // _setmode, _isatty +# ifdef __MINGW32__ + int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h> +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include <unistd.h> // isatty +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + +//************************************** +// Compiler-specific functions +//************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if defined(_MSC_VER) // Visual Studio +# define swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define swap32 __builtin_bswap32 +#else + static inline unsigned int swap32(unsigned int x) + { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); + } +#endif + + +//**************************** +// Constants +//**************************** +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define MAGICNUMBER_SIZE 4 +#define LZ4S_MAGICNUMBER 0x184D2204 +#define LZ4S_SKIPPABLE0 0x184D2A50 +#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 +#define LEGACY_MAGICNUMBER 0x184C2102 + +#define CACHELINE 64 +#define LEGACY_BLOCKSIZE (8 MB) +#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) +#define LZ4S_BLOCKSIZEID_DEFAULT 7 +#define LZ4S_CHECKSUM_SEED 0 +#define LZ4S_EOS 0 +#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) + + +//************************************** +// Architecture Macros +//************************************** +static const int one = 1; +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) + + +//************************************** +// Macros +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + +//************************************** +// Local Parameters +//************************************** +static int displayLevel = 0; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information +static int overwrite = 1; +static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; +static int blockChecksum = 0; +static int streamChecksum = 1; +static int blockIndependence = 1; + +static const int minBlockSizeID = 4; +static const int maxBlockSizeID = 7; + +//************************************** +// Exceptions +//************************************** +#define DEBUG 0 +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +//************************************** +// Version modifiers +//************************************** +#define EXTENDED_ARGUMENTS +#define EXTENDED_HELP +#define EXTENDED_FORMAT +#define DEFAULT_COMPRESSOR compress_file +#define DEFAULT_DECOMPRESSOR decodeLZ4S + + +/* ************************************************** */ +/* ****************** Parameters ******************** */ +/* ************************************************** */ + +/* Default setting : overwrite = 1; return : overwrite mode (0/1) */ +int LZ4IO_setOverwrite(int yes) +{ + overwrite = (yes!=0); + return overwrite; +} + +/* blockSizeID : valid values : 4-5-6-7 */ +int LZ4IO_setBlockSizeID(int bsid) +{ + static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; + if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1; + blockSizeId = bsid; + return blockSizeTable[blockSizeId-minBlockSizeID]; +} + + +int LZ4IO_setBlockMode(blockMode_t blockMode) +{ + blockIndependence = (blockMode == independentBlocks); + return blockIndependence; +} + + +/* Default setting : no checksum */ +int LZ4IO_setBlockChecksumMode(int xxhash) +{ + blockChecksum = (xxhash != 0); + return blockChecksum; +} + + +/* Default setting : checksum enabled */ +int LZ4IO_setStreamChecksumMode(int xxhash) +{ + streamChecksum = (xxhash != 0); + return streamChecksum; +} + + +/* Default setting : 0 (no notification) */ +int LZ4IO_setNotificationLevel(int level) +{ + displayLevel = level; + return displayLevel; +} + + + +/* ************************************************************************ */ +/* ********************** LZ4 File / Stream compression ******************* */ +/* ************************************************************************ */ + +static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } +static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } +static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } + + +static int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) +{ + + if (!strcmp (input_filename, stdinmark)) + { + DISPLAYLEVEL(4,"Using stdin for input\n"); + *pfinput = stdin; + SET_BINARY_MODE(stdin); + } + else + { + *pfinput = fopen(input_filename, "rb"); + } + + if (!strcmp (output_filename, stdoutmark)) + { + DISPLAYLEVEL(4,"Using stdout for output\n"); + *pfoutput = stdout; + SET_BINARY_MODE(stdout); + } + else + { + // Check if destination file already exists + *pfoutput=0; + if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); + if (*pfoutput!=0) + { + fclose(*pfoutput); + if (!overwrite) + { + char ch; + DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); + DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); + if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible + ch = (char)getchar(); + if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); + } + } + *pfoutput = fopen( output_filename, "wb" ); + } + + if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); + if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); + + return 0; +} + + +// LZ4IO_compressFilename_Legacy : This function is "hidden" (not published in .h) +// Its purpose is to generate compressed streams using the old 'legacy' format +int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = MAGICNUMBER_SIZE; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int displayLevel = (compressionlevel>0); + clock_t start, end; + size_t sizeCheck; + + + // Init + if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + + // Allocate Memory + in_buff = (char*)malloc(LEGACY_BLOCKSIZE); + out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); + sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); + + // Main Loop + while (1) + { + unsigned int outSize; + // Read Block + int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + if( inSize<=0 ) break; + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, inSize); + compressedfilesize += outSize+4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) +{ + void* (*initFunction) (const char*); + int (*compressionFunction)(void*, const char*, char*, int, int); + char* (*translateFunction) (void*); + int (*freeFunction) (void*); + void* ctx; + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff, *in_start, *in_end; + char* out_buff; + FILE* finput; + FILE* foutput; + clock_t start, end; + unsigned int blockSize, inputBufferSize; + size_t sizeCheck, header_size; + void* streamChecksumState=NULL; + + + // Init + start = clock(); + if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; + if (compressionlevel>=3) + { + initFunction = LZ4_createHC; + compressionFunction = LZ4_compressHC_limitedOutput_continue; + translateFunction = LZ4_slideInputBufferHC; + freeFunction = LZ4_freeHC; + } + else + { + initFunction = LZ4_create; + compressionFunction = LZ4_compress_limitedOutput_continue; + translateFunction = LZ4_slideInputBuffer; + freeFunction = LZ4_free; + } + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + inputBufferSize = blockSize + 64 KB; + if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE; + in_buff = (char*)malloc(inputBufferSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); + in_start = in_buff; in_end = in_buff + inputBufferSize; + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + ctx = initFunction(in_buff); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') + *(out_buff+4) |= (blockIndependence & _1BIT) << 5; + *(out_buff+4) |= (blockChecksum & _1BIT) << 4; + *(out_buff+4) |= (streamChecksum & _1BIT) << 2; + *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(out_buff+6) = (unsigned char) checkbits; + header_size = 7; + sizeCheck = fwrite(out_buff, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // Main Loop + while (1) + { + unsigned int outSize; + unsigned int inSize; + // Read Block + if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); + inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); + if( inSize==0 ) break; // No more input : end of compression + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); + + // Compress Block + outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + + } + else // Copy Original + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_start, 1, inSize, foutput); + if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + in_start += inSize; + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + freeFunction(ctx); + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +FORCE_INLINE int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel) +{ (void)clevel; return LZ4_compress_limitedOutput(src, dst, size, maxOut); } + +int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel) +{ + int (*compressionFunction)(const char*, char*, int, int, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff; + char* out_buff; + char* headerBuffer; + FILE* finput; + FILE* foutput; + clock_t start, end; + int blockSize; + size_t sizeCheck, header_size, readSize; + void* streamChecksumState=NULL; + + // Branch out + if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionLevel); + + // Init + start = clock(); + if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; + if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local; + else { compressionFunction = LZ4_compressHC2_limitedOutput; } + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + in_buff = (char*)malloc(blockSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); + if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Write Archive Header + *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') + *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; + *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; + *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; + *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(headerBuffer+6) = (unsigned char) checkbits; + header_size = 7; + + // Write header + sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // read first block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + + // Main Loop + while (readSize>0) + { + unsigned int outSize; + + filesize += readSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + } + else // Copy Original Uncompressed + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_buff, 1, readSize, foutput); + if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + + // Read next block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Close & Free + free(in_buff); + free(out_buff); + free(headerBuffer); + fclose(finput); + fclose(foutput); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + return 0; +} + + +/* ********************************************************************* */ +/* ********************** LZ4 File / Stream decoding ******************* */ +/* ********************************************************************* */ + +static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff; + unsigned int blockSize; + + + // Allocate Memory + in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + out_buff = (char*)malloc(LEGACY_BLOCKSIZE); + if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); + + // Main Loop + while (1) + { + int decodeSize; + size_t sizeCheck; + + // Block Size + sizeCheck = fread(&blockSize, 1, 4, finput); + if (sizeCheck==0) break; // Nothing to read : file read is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian + if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) + { // Cannot read next block : maybe new stream ? + fseek(finput, -4, SEEK_CUR); + break; + } + + // Read Block + sizeCheck = fread(in_buff, 1, blockSize, finput); + + // Decode Block + decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); + filesize += decodeSize; + + // Write Block + sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); + if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff, *out_start, *out_end; + unsigned char descriptor[LZ4S_MAXHEADERSIZE]; + size_t nbReadBytes; + int decodedBytes=0; + unsigned int maxBlockSize; + size_t sizeCheck; + int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; + void* streamChecksumState=NULL; + int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; + unsigned int prefix64k = 0; + + // Decode stream descriptor + nbReadBytes = fread(descriptor, 1, 3, finput); + if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); + { + int version = (descriptor[0] >> 6) & _2BITS; + int streamSize = (descriptor[0] >> 3) & _1BIT; + int reserved1 = (descriptor[0] >> 1) & _1BIT; + int dictionary = (descriptor[0] >> 0) & _1BIT; + + int reserved2 = (descriptor[1] >> 7) & _1BIT; + int blockSizeId = (descriptor[1] >> 4) & _3BITS; + int reserved3 = (descriptor[1] >> 0) & _4BITS; + int checkBits = (descriptor[2] >> 0) & _8BITS; + int checkBits_xxh32; + + blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; + blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; + streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; + + if (version != 1) EXM_THROW(62, "Wrong version number"); + if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); + if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); + if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); + if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); + if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); + // Checkbits verification + descriptor[1] &= 0xF0; + checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); + checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); + if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); + } + + if (!blockIndependenceFlag) + { + decompressionFunction = LZ4_decompress_safe_withPrefix64k; + prefix64k = 64 KB; + } + + // Allocate Memory + { + unsigned int outbuffSize = prefix64k+maxBlockSize; + in_buff = (char*)malloc(maxBlockSize); + if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; + out_buff = (char*)malloc(outbuffSize); + out_end = out_buff + outbuffSize; + out_start = out_buff + prefix64k; + if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); + } + if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Main Loop + while (1) + { + unsigned int blockSize, uncompressedFlag; + + // Block Size + nbReadBytes = fread(&blockSize, 1, 4, finput); + if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); + if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian + uncompressedFlag = blockSize >> 31; + blockSize &= 0x7FFFFFFF; + if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); + + // Read Block + nbReadBytes = fread(in_buff, 1, blockSize, finput); + if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); + + // Check Block + if (blockChecksumFlag) + { + unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); + } + + if (uncompressedFlag) + { + // Write uncompressed Block + sizeCheck = fwrite(in_buff, 1, blockSize, foutput); + if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); + filesize += blockSize; + if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); + if (!blockIndependenceFlag) + { + if (blockSize >= prefix64k) + { + memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks + out_start = out_buff + prefix64k; + continue; + } + else + { + memcpy(out_start, in_buff, blockSize); + decodedBytes = blockSize; + } + } + } + else + { + // Decode Block + decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); + if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); + filesize += decodedBytes; + if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); + + // Write Block + sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); + if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); + } + + if (!blockIndependenceFlag) + { + out_start += decodedBytes; + if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) + { + memcpy(out_buff, out_start - prefix64k, prefix64k); + out_start = out_buff + prefix64k; + } + } + } + + // Stream Checksum + if (streamChecksumFlag) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +static unsigned long long selectDecoder( FILE* finput, FILE* foutput) +{ + unsigned int magicNumber, size; + int errorNb; + size_t nbReadBytes; + + // Check Archive Header + nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) return 0; // EOF + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); + magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format + if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers + + switch(magicNumber) + { + case LZ4S_MAGICNUMBER: + return DEFAULT_DECOMPRESSOR(finput, foutput); + case LEGACY_MAGICNUMBER: + DISPLAYLEVEL(4, "Detected : Legacy format \n"); + return decodeLegacyStream(finput, foutput); + case LZ4S_SKIPPABLE0: + DISPLAYLEVEL(4, "Skipping detected skippable area \n"); + nbReadBytes = fread(&size, 1, 4, finput); + if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); + size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format + errorNb = fseek(finput, size, SEEK_CUR); + if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); + return selectDecoder(finput, foutput); + EXTENDED_FORMAT; + default: + if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream + DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); + return 0; + } +} + + +int LZ4IO_decompressFilename(char* input_filename, char* output_filename) +{ + unsigned long long filesize = 0, decodedSize=0; + FILE* finput; + FILE* foutput; + clock_t start, end; + + + // Init + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + + // Loop over multiple streams + do + { + decodedSize = selectDecoder(finput, foutput); + filesize += decodedSize; + } while (decodedSize); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close + fclose(finput); + fclose(foutput); + + // Error status = OK + return 0; +} + diff --git a/programs/lz4io.h b/programs/lz4io.h new file mode 100644 index 0000000..cb35ec0 --- /dev/null +++ b/programs/lz4io.h @@ -0,0 +1,77 @@ +/* + LZ4io.h - LZ4 File/Stream Interface + Copyright (C) Yann Collet 2011-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* + Note : this is stand-alone program. + It is not part of LZ4 compression library, it is a user code of the LZ4 library. + - The license of LZ4 library is BSD. + - The license of xxHash library is BSD. + - The license of this source file is GPLv2. +*/ + + +/* ************************************************** */ +/* Special input/output values */ +/* ************************************************** */ +#define NULL_OUTPUT "null" +static char stdinmark[] = "stdin"; +static char stdoutmark[] = "stdout"; +#ifdef _WIN32 +static char nulmark[] = "nul"; +#else +static char nulmark[] = "/dev/null"; +#endif + + +/* ************************************************** */ +/* ****************** Functions ********************* */ +/* ************************************************** */ + +int LZ4IO_compressFilename (char* input_filename, char* output_filename, int compressionlevel); +int LZ4IO_decompressFilename(char* input_filename, char* output_filename); + + +/* ************************************************** */ +/* ****************** Parameters ******************** */ +/* ************************************************** */ + +/* Default setting : overwrite = 1; + return : overwrite mode (0/1) */ +int LZ4IO_setOverwrite(int yes); + +/* blockSizeID : valid values : 4-5-6-7 + return : -1 if error, blockSize if OK */ +int LZ4IO_setBlockSizeID(int blockSizeID); + +/* Default setting : independent blocks */ +typedef enum { chainedBlocks, independentBlocks } blockMode_t; +int LZ4IO_setBlockMode(blockMode_t blockMode); + +/* Default setting : no checksum */ +int LZ4IO_setBlockChecksumMode(int xxhash); + +/* Default setting : checksum enabled */ +int LZ4IO_setStreamChecksumMode(int xxhash); + +/* Default setting : 0 (no notification) */ +int LZ4IO_setNotificationLevel(int level); |