From b3b207c4db9378f3eb3518c0778ff03a9b5cdbb3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 23 Nov 2014 00:46:15 +0100 Subject: New endian & alignment code --- NEWS | 10 +- lz4.c | 420 +++++++++++++++++++++++++++++++++------------------------------ lz4.h | 12 +- lz4hc.h | 6 +- xxhash.c | 72 +++++------ 5 files changed, 275 insertions(+), 245 deletions(-) diff --git a/NEWS b/NEWS index 7edee54..763fa5a 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,12 @@ +r125: +Changed : endian and alignment code +Fixed : some alignment warnings under clang + r124: -Fix : LZ4F_compressBound() using NULL preferencesPtr -Updated : xxHash, to r37 +New : LZ4 HC streaming mode +Fixed : LZ4F_compressBound() using null preferencesPtr +Updated : xxHash to r38 +Updated library number, to 1.4.0 r123: Added : experimental lz4frame API, thanks to Takayuki Matsuoka and Christopher Jackson for testings diff --git a/lz4.c b/lz4.c index 198b581..579a4fc 100644 --- a/lz4.c +++ b/lz4.c @@ -31,63 +31,37 @@ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ + /************************************** Tuning parameters **************************************/ /* * HEAPMODE : * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)). + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ #define HEAPMODE 0 +/* + * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : + * You can force the code to use unaligned memory access if you know your CPU can handle it. + */ +/* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ + /************************************** CPU Feature Detection **************************************/ -/* 32 or 64 bits ? */ -#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__64BIT__) || defined(__mips64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ - || defined(__s390x__) ) /* Detects 64 bits mode */ -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 -#endif -#define LZ4_32BITS (sizeof(void*)==4) -#define LZ4_64BITS (sizeof(void*)==8) - /* - * Little Endian or Big Endian ? - * Overwrite the #define below if you know your architecture endianess + * Unaligned memory access detection */ -#include /* Apparently required to detect endianess */ -#if defined (__GLIBC__) -# include -# if (__BYTE_ORDER == __BIG_ENDIAN) -# define LZ4_BIG_ENDIAN 1 -# endif -#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) -# define LZ4_BIG_ENDIAN 1 -#elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) -# define LZ4_BIG_ENDIAN 1 +#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ + || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(__i386__) || defined(__x86_64__) \ + || defined(_M_IX86) || defined(_M_X64) +# define LZ4_UNALIGNED_ACCESS 1 #else -/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ -#endif - -/* - * Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance - */ -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 +# define LZ4_UNALIGNED_ACCESS 0 #endif /* Define this parameter if your target system or compiler does not support hardware bit count */ @@ -95,18 +69,9 @@ # define LZ4_FORCE_SW_BITCOUNT #endif -/* - * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : - * This option may provide a small boost to performance for some big endian cpu, although probably modest. - * You may set this option to 1 if data will remain within closed environment. - * This option is useless on Little_Endian CPU (such as x86) - */ - -/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */ - /************************************** - Compiler Options + Compiler Options **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ /* "restrict" is a known keyword */ @@ -117,13 +82,6 @@ #ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline # include /* For Visual 2005 */ -# if LZ4_ARCH64 /* 64-bits */ -# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ -# else /* 32-bits */ -# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ -# endif # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #else # ifdef __GNUC__ @@ -133,12 +91,6 @@ # endif #endif -#ifdef _MSC_VER /* Visual Studio */ -# define lz4_bswap16(x) _byteswap_ushort(x) -#else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) @@ -185,37 +137,167 @@ typedef unsigned long long U64; #endif -#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif +/************************************** + Reading and writing into memory +**************************************/ +static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } -typedef struct { U16 v; } _PACKED U16_S; -typedef struct { U32 v; } _PACKED U32_S; -typedef struct { U64 v; } _PACKED U64_S; -typedef struct {size_t v;} _PACKED size_t_S; +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) -# pragma pack(0) -# else -# pragma pack(pop) -# endif -#endif +static U16 LZ4_readLE16(const void* memPtr) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + return *(U16*)memPtr; + { + const BYTE* p = memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + { + *(U16*)memPtr = value; + return; + } + { + BYTE* p = memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +static U32 LZ4_readLE32(const void* memPtr) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + return *(U32*)memPtr; + { + const BYTE* p = memPtr; + U32 result = (U32)((U32)p[0] + (p[1]<<8) + (p[2]<<16) + ((U32)p[3]<<24)); + return result; + } +} + +/* +static void LZ4_writeLE32(void* memPtr, U32 value) +{ + BYTE* p = memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + p[2] = (BYTE)(value>>16); + p[3] = (BYTE)(value>>24); +} +*/ + +static void LZ4_copy4(void* dstPtr, const void* srcPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + { + *(U32*)dstPtr = *(U32*)srcPtr; + return; + } + { + BYTE* d = dstPtr; + const BYTE* s = srcPtr; + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = s[3]; + } +} + +static U64 LZ4_readLE64(const void* memPtr) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + return *(U64*)memPtr; + { + const BYTE* p = memPtr; + return (U64)((U64)p[0] + (p[1]<<8) + (p[2]<<16) + ((U64)p[3]<<24) + + (((U64)p[4])<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); + } +} + +/* +static void LZ4_writeLE64(void* memPtr, U64 value) +{ + BYTE* p = memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + p[2] = (BYTE)(value>>16); + p[3] = (BYTE)(value>>24); + p[4] = (BYTE)(value>>32); + p[5] = (BYTE)(value>>40); + p[6] = (BYTE)(value>>48); + p[7] = (BYTE)(value>>56); +} +*/ + +static void LZ4_copy8(void* dstPtr, const void* srcPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + { + if (LZ4_64bits()) + *(U64*)dstPtr = *(U64*)srcPtr; + else + ((U32*)dstPtr)[0] = ((U32*)srcPtr)[0], + ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; + return; + } + { + BYTE* d = dstPtr; + const BYTE* s = srcPtr; + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = s[3]; + d[4] = s[4]; + d[5] = s[5]; + d[6] = s[6]; + d[7] = s[7]; + } +} -#define A16(x) (((U16_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -#define AARCH(x) (((size_t_S *)(x))->v) +#define STEPSIZE sizeof(size_t) + +static size_t LZ4_readLE_ARCH(const void* p) +{ + if (LZ4_64bits()) + return (size_t)LZ4_readLE64(p); + else + return (size_t)LZ4_readLE32(p); +} + +/* +static void LZ4_writeLE_ARCH(void* p, size_t value) +{ + if (LZ4_64BITS) + LZ4_writeLE64(p, (U64)value); + else + LZ4_writeLE32(p, (U32)value); +} + +static void LZ4_copyARCH(void* dstPtr, const void* srcPtr) +{ + if (LZ4_64BITS) + LZ4_copy8(dstPtr, srcPtr); + else + LZ4_copy4(dstPtr, srcPtr); +} +*/ + +#if !defined(__GNUC__) +# define LZ4_WILDCOPY(d,s,e) { do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=e; */ +#else +# define LZ4_WILDCOPY64(d,s,e) { do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=e; */ +# define LZ4_WILDCOPY32(d,s,e) { if (likely(e-d <= 8)) { LZ4_copy8(d,s); d+=8; s+=8; } else do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=e; */ -#else -# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif -# else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; - _BitScanForward64( &r, val ); + _BitScanForward64( &r, (U64)val ); return (int)(r>>3); # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); + return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif -# endif -} - -#else - -static int LZ4_NbCommonBytes (register U32 val) -{ -# if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif -# else + } + /* 32 bits */ + { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r; - _BitScanForward( &r, val ); + _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); + return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif -# endif + } } -#endif - - -/******************************** - Compression functions -********************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -static int LZ4_hashSequence(U32 sequence, tableType_t tableType) +static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType) { if (tableType == byU16) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -380,15 +403,15 @@ static int LZ4_hashSequence(U32 sequence, tableType_t tableType) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); } +static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_readLE32(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { switch (tableType) { - case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; } + case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } } @@ -417,13 +440,13 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimi while (likely(pIn=lowRefLimit) : 1) && (ref+MAX_DISTANCE>=ip) - && (A32(ref+refDelta)==A32(ip)) ) + && (LZ4_readLE32(ref+refDelta)==LZ4_readLE32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ @@ -646,16 +669,16 @@ _last_literals: int LZ4_compress(const char* source, char* dest, int inputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */ + void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ #else - U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ + U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ #endif int result; if (inputSize < (int)LZ4_64KLIMIT) result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); #if (HEAPMODE) FREEMEM(ctx); @@ -666,16 +689,16 @@ int LZ4_compress(const char* source, char* dest, int inputSize) int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */ + void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 4); /* Aligned on 8-bytes boundaries */ #else - U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */ + U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ #endif int result; if (inputSize < (int)LZ4_64KLIMIT) result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); #if (HEAPMODE) FREEMEM(ctx); @@ -700,7 +723,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(4, LZ4_STREAMSIZE_U32); + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ LZ4_resetStream(lz4s); return lz4s; @@ -959,11 +982,12 @@ FORCE_INLINE int LZ4_decompress_generic( LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; /* get offset */ - LZ4_READ_LITTLEENDIAN_16(match,cpy,ip); ip+=2; + match = cpy - LZ4_readLE16(ip); ip+=2; if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ /* get matchlength */ - if ((length=(token&ML_MASK)) == ML_MASK) + length = token & ML_MASK; + if (length == ML_MASK) { unsigned s; do @@ -1020,9 +1044,9 @@ FORCE_INLINE int LZ4_decompress_generic( op[2] = match[2]; op[3] = match[3]; match += dec32table[op-match]; - A32(op+4) = A32(match); + LZ4_copy4(op+4, match); op += 8; match -= dec64; - } else { LZ4_COPY8(op,match); } + } else { LZ4_copy8(op, match); op+=8; match+=8; } if (unlikely(cpy>oend-12)) { @@ -1079,7 +1103,7 @@ typedef struct */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64); return lz4s; } @@ -1241,7 +1265,7 @@ int LZ4_resetStreamState(void* state, const char* inputBuffer) void* LZ4_create (const char* inputBuffer) { - void* lz4ds = ALLOCATOR(4, LZ4_STREAMSIZE_U32); + void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); return lz4ds; } @@ -1267,7 +1291,7 @@ int LZ4_compress_withState (void* state, const char* source, char* dest, int inp if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); } int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) @@ -1278,7 +1302,7 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue); + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); } /* Obsolete streaming decompression functions */ diff --git a/lz4.h b/lz4.h index 0350f6a..f995b05 100644 --- a/lz4.h +++ b/lz4.h @@ -172,14 +172,14 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS Experimental Streaming Compression Functions ***********************************************/ -#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int)) +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) /* * LZ4_stream_t * information structure to track an LZ4 stream. * important : init this structure content before first use ! */ -typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t; +typedef struct { unsigned long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; /* * LZ4_resetStream @@ -234,14 +234,14 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); Experimental Streaming Decompression Functions ************************************************/ -#define LZ4_STREAMDECODESIZE_U32 8 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int)) +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) /* * LZ4_streamDecode_t * information structure to track an LZ4 stream. * important : init this structure content using LZ4_setStreamDecode or memset() before first use ! */ -typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t; +typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; /* * If you prefer dynamic allocation methods, diff --git a/lz4hc.h b/lz4hc.h index 23ad7c1..26ba6d3 100644 --- a/lz4hc.h +++ b/lz4hc.h @@ -105,9 +105,9 @@ They just use the externally allocated memory for state instead of allocating th /************************************** Experimental Streaming Functions **************************************/ -#define LZ4_STREAMHCSIZE_U32 65548 -#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U32 * sizeof(unsigned int)) -typedef struct { unsigned int table[LZ4_STREAMHCSIZE_U32]; } LZ4_streamHC_t; +#define LZ4_STREAMHCSIZE_U64 32774 +#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ4_STREAMHCSIZE_U64]; } LZ4_streamHC_t; /* This structure allows static allocation of LZ4 HC streaming state. diff --git a/xxhash.c b/xxhash.c index e6c2f31..24a64b5 100644 --- a/xxhash.c +++ b/xxhash.c @@ -84,11 +84,11 @@ You can contact the author at : // Modify the local functions below should you wish to use some other memory routines // for malloc(), free() #include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } // for memcpy() #include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } @@ -221,28 +221,28 @@ static const int one = 1; //**************************** typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); + return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); } -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } -FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_alignment align) +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); else - return endian==XXH_littleEndian ? *ptr : XXH_swap64(*ptr); + return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); } -FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } @@ -256,7 +256,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) #ifdef XXH_ACCEPT_NULL_INPUT_POINTER if (p==NULL) @@ -361,7 +361,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) #ifdef XXH_ACCEPT_NULL_INPUT_POINTER if (p==NULL) @@ -509,8 +509,8 @@ typedef struct U32 v2; U32 v3; U32 v4; + U32 mem32[4]; /* defined as U32 for alignment */ U32 memsize; - char memory[16]; } XXH_istate32_t; typedef struct @@ -521,8 +521,8 @@ typedef struct U64 v2; U64 v3; U64 v4; + U64 mem64[4]; /* defined as U64 for alignment */ U32 memsize; - char memory[32]; } XXH_istate64_t; @@ -592,16 +592,16 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v if (state->memsize + len < 16) // fill in tmp buffer { - XXH_memcpy(state->memory + state->memsize, input, len); + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } if (state->memsize) // some data left from previous update { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); { - const U32* p32 = (const U32*)state->memory; + const U32* p32 = state->mem32; state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; @@ -633,19 +633,19 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v do { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v1 += XXH_readLE32(p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v2 += XXH_readLE32(p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v3 += XXH_readLE32(p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v4 += XXH_readLE32(p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; @@ -660,7 +660,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v if (p < bEnd) { - XXH_memcpy(state->memory, p, bEnd-p); + XXH_memcpy(state->mem32, p, bEnd-p); state->memsize = (int)(bEnd-p); } @@ -682,8 +682,8 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { XXH_istate32_t* state = (XXH_istate32_t*) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; + const BYTE * p = (const BYTE*)state->mem32; + BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->total_len >= 16) @@ -699,7 +699,7 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endiane while (p+4<=bEnd) { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 += XXH_readLE32(p, endian) * PRIME32_3; h32 = XXH_rotl32(h32, 17) * PRIME32_4; p+=4; } @@ -746,16 +746,16 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v if (state->memsize + len < 32) // fill in tmp buffer { - XXH_memcpy(state->memory + state->memsize, input, len); + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } if (state->memsize) // some data left from previous update { - XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); { - const U64* p64 = (const U64*)state->memory; + const U64* p64 = state->mem64; state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; state->v1 = XXH_rotl64(state->v1, 31); state->v1 *= PRIME64_1; @@ -787,19 +787,19 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v do { - v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v1 += XXH_readLE64(p, endian) * PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; p+=8; - v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v2 += XXH_readLE64(p, endian) * PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; p+=8; - v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v3 += XXH_readLE64(p, endian) * PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; p+=8; - v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v4 += XXH_readLE64(p, endian) * PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; p+=8; @@ -814,7 +814,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v if (p < bEnd) { - XXH_memcpy(state->memory, p, bEnd-p); + XXH_memcpy(state->mem64, p, bEnd-p); state->memsize = (int)(bEnd-p); } @@ -836,8 +836,8 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { XXH_istate64_t * state = (XXH_istate64_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; + const BYTE * p = (const BYTE*)state->mem64; + BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) @@ -882,7 +882,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endiane while (p+8<=bEnd) { - U64 k1 = XXH_readLE64((const U64*)p, endian); + U64 k1 = XXH_readLE64(p, endian); k1 *= PRIME64_2; k1 = XXH_rotl64(k1,31); k1 *= PRIME64_1; @@ -893,7 +893,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endiane if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p+=4; } -- cgit v0.12 From 2ad37dbe58eb3709e5dee16ebcf345bd53680f85 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 23 Nov 2014 01:14:04 +0100 Subject: Corrected lz4io --- lz4.c | 11 +++++++++-- lz4.h | 2 +- programs/lz4cli.c | 2 +- programs/lz4io.c | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lz4.c b/lz4.c index 579a4fc..7e454fd 100644 --- a/lz4.c +++ b/lz4.c @@ -44,7 +44,7 @@ /* * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : - * You can force the code to use unaligned memory access if you know your CPU can handle it. + * You can force the code to use unaligned memory access if you know your CPU can handle it efficiently. */ /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ @@ -58,7 +58,9 @@ #if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ || defined(__ARM_FEATURE_UNALIGNED) \ || defined(__i386__) || defined(__x86_64__) \ - || defined(_M_IX86) || defined(_M_X64) + || defined(_M_IX86) || defined(_M_X64) \ + || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ + || (defined(_M_ARM) && (_M_ARM >= 7)) # define LZ4_UNALIGNED_ACCESS 1 #else # define LZ4_UNALIGNED_ACCESS 0 @@ -153,6 +155,7 @@ static U16 LZ4_readLE16(const void* memPtr) { if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) return *(U16*)memPtr; + else { const BYTE* p = memPtr; return (U16)((U16)p[0] + (p[1]<<8)); @@ -166,6 +169,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) *(U16*)memPtr = value; return; } + else { BYTE* p = memPtr; p[0] = (BYTE) value; @@ -202,6 +206,7 @@ static void LZ4_copy4(void* dstPtr, const void* srcPtr) *(U32*)dstPtr = *(U32*)srcPtr; return; } + else { BYTE* d = dstPtr; const BYTE* s = srcPtr; @@ -216,6 +221,7 @@ static U64 LZ4_readLE64(const void* memPtr) { if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) return *(U64*)memPtr; + else { const BYTE* p = memPtr; return (U64)((U64)p[0] + (p[1]<<8) + (p[2]<<16) + ((U64)p[3]<<24) + @@ -249,6 +255,7 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr) ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; return; } + else { BYTE* d = dstPtr; const BYTE* s = srcPtr; diff --git a/lz4.h b/lz4.h index f995b05..c9ed49f 100644 --- a/lz4.h +++ b/lz4.h @@ -48,7 +48,7 @@ extern "C" { **************************************/ #define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ #define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 2d612e7..9a6e5bc 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -113,7 +113,7 @@ //**************************** #define COMPRESSOR_NAME "LZ4 Compression CLI" #ifndef LZ4_VERSION -# define LZ4_VERSION "r122" +# define LZ4_VERSION "r125" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ diff --git a/programs/lz4io.c b/programs/lz4io.c index 3a84866..afaa59f 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -357,7 +357,7 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i static void* LZ4IO_LZ4_createStream (const char* inputBuffer) { (void)inputBuffer; - return calloc(4, LZ4_STREAMSIZE_U32); + return calloc(8, LZ4_STREAMSIZE_U64); } static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) -- cgit v0.12 From 86fa21b3019d7e9983704661966a64f5aa1456bf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 23 Nov 2014 18:36:04 +0100 Subject: Added : $(EXT) within install for cross-compilation support --- Makefile | 9 +-------- lz4.c | 5 ++++- programs/Makefile | 14 +++++++------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 586ac5e..c2bbd57 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ # ################################################################ # Version numbers -VERSION=124 +VERSION=125 export RELEASE=r$(VERSION) LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` @@ -50,13 +50,6 @@ PRGDIR = programs DISTRIBNAME=lz4-$(RELEASE).tar.gz -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -else -EXT = -endif - # OS X linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html ifeq ($(shell uname), Darwin) diff --git a/lz4.c b/lz4.c index 7e454fd..8e81cc2 100644 --- a/lz4.c +++ b/lz4.c @@ -44,7 +44,10 @@ /* * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : - * You can force the code to use unaligned memory access if you know your CPU can handle it efficiently. + * You can force the code to use unaligned memory access, should you know your CPU can handle it efficiently. + * If it effectively results in better speed (up to 50% improvement can be expected) + * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) + * so that an automatic detection macro can be added to mainline. */ /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ diff --git a/programs/Makefile b/programs/Makefile index 1d7e17f..fcfb32c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE=r124 +RELEASE=r125 DESTDIR?= PREFIX ?= /usr @@ -113,9 +113,9 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) install: lz4 lz4c @echo Installing binaries @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ - @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 - @ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat - @install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c + @install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT) + @ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat + @install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT) @echo Installing man pages @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 @install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1 @@ -124,12 +124,12 @@ install: lz4 lz4c uninstall: rm -f $(DESTDIR)$(BINDIR)/lz4cat - [ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4 - [ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c + [ -x $(DESTDIR)$(BINDIR)/lz4$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4$(EXT) + [ -x $(DESTDIR)$(BINDIR)/lz4c$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4c$(EXT) [ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1 [ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1 [ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 - @echo lz4 successfully uninstalled + @echo lz4 programs successfully uninstalled test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem -- cgit v0.12 From 2d98faf238516ecd6588f9202f8ae546c0b798e4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 27 Nov 2014 22:44:36 +0100 Subject: Improved decoding speed --- lz4.c | 181 ++++++++++++++++++++++----------------------------- lz4.h | 4 +- programs/fullbench.c | 20 +++++- 3 files changed, 98 insertions(+), 107 deletions(-) diff --git a/lz4.c b/lz4.c index 8e81cc2..b3f1b2d 100644 --- a/lz4.c +++ b/lz4.c @@ -56,7 +56,9 @@ CPU Feature Detection **************************************/ /* - * Unaligned memory access detection + * Automated efficient unaligned memory access detection + * Based on known hardware architectures + * This list will be updated thanks to Open Source community feedbacks */ #if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ || defined(__ARM_FEATURE_UNALIGNED) \ @@ -146,6 +148,8 @@ /************************************** Reading and writing into memory **************************************/ +#define STEPSIZE sizeof(size_t) + static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } static unsigned LZ4_isLittleEndian(void) @@ -180,43 +184,44 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } -static U32 LZ4_readLE32(const void* memPtr) + +static U32 LZ4_read16(const void* memPtr) { - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) - return *(U32*)memPtr; + if (LZ4_UNALIGNED_ACCESS) { - const BYTE* p = memPtr; - U32 result = (U32)((U32)p[0] + (p[1]<<8) + (p[2]<<16) + ((U32)p[3]<<24)); - return result; + return *(U16*)memPtr; + } + else + { + U16 val16; + memcpy(&val16, memPtr, 2); + return val16; } } -/* -static void LZ4_writeLE32(void* memPtr, U32 value) -{ - BYTE* p = memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - p[2] = (BYTE)(value>>16); - p[3] = (BYTE)(value>>24); -} -*/ - -static void LZ4_copy4(void* dstPtr, const void* srcPtr) +static U32 LZ4_read32(const void* memPtr) { if (LZ4_UNALIGNED_ACCESS) { - *(U32*)dstPtr = *(U32*)srcPtr; - return; + return *(U32*)memPtr; } else { - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - d[0] = s[0]; - d[1] = s[1]; - d[2] = s[2]; - d[3] = s[3]; + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; + } +} + + +static U32 LZ4_readLE32(const void* memPtr) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + return *(U32*)memPtr; + { + const BYTE* p = memPtr; + U32 result = (U32)((U32)p[0] + (p[1]<<8) + (p[2]<<16) + ((U32)p[3]<<24)); + return result; } } @@ -232,20 +237,24 @@ static U64 LZ4_readLE64(const void* memPtr) } } -/* -static void LZ4_writeLE64(void* memPtr, U64 value) +static size_t LZ4_readLE_ARCH(const void* p) +{ + if (LZ4_64bits()) + return (size_t)LZ4_readLE64(p); + else + return (size_t)LZ4_readLE32(p); +} + + +static void LZ4_copy4(void* dstPtr, const void* srcPtr) { - BYTE* p = memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - p[2] = (BYTE)(value>>16); - p[3] = (BYTE)(value>>24); - p[4] = (BYTE)(value>>32); - p[5] = (BYTE)(value>>40); - p[6] = (BYTE)(value>>48); - p[7] = (BYTE)(value>>56); + if (LZ4_UNALIGNED_ACCESS) + { + *(U32*)dstPtr = *(U32*)srcPtr; + return; + } + memcpy(dstPtr, srcPtr, 4); } -*/ static void LZ4_copy8(void* dstPtr, const void* srcPtr) { @@ -258,57 +267,18 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr) ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; return; } - else - { - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - d[0] = s[0]; - d[1] = s[1]; - d[2] = s[2]; - d[3] = s[3]; - d[4] = s[4]; - d[5] = s[5]; - d[6] = s[6]; - d[7] = s[7]; - } + memcpy(dstPtr, srcPtr, 8); } -#define STEPSIZE sizeof(size_t) - -static size_t LZ4_readLE_ARCH(const void* p) +/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - if (LZ4_64bits()) - return (size_t)LZ4_readLE64(p); - else - return (size_t)LZ4_readLE32(p); + BYTE* d = dstPtr; + const BYTE* s = srcPtr; + BYTE* e = dstEnd; + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=e; */ -#else -# define LZ4_WILDCOPY64(d,s,e) { do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=e; */ -# define LZ4_WILDCOPY32(d,s,e) { if (likely(e-d <= 8)) { LZ4_copy8(d,s); d+=8; s+=8; } else do { LZ4_copy8(d,s); d+=8; s+=8; } while (d> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_readLE32(p), tableType); } +static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { @@ -444,20 +414,20 @@ static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t t return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimit) +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; while (likely(pIn=lowRefLimit) : 1) && (ref+MAX_DISTANCE>=ip) - && (LZ4_readLE32(ref+refDelta)==LZ4_readLE32(ip)) ) + && (LZ4_read32(ref+refDelta)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ @@ -699,7 +669,7 @@ int LZ4_compress(const char* source, char* dest, int inputSize) int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 4); /* Aligned on 8-bytes boundaries */ + void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ #else U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ #endif @@ -989,7 +959,9 @@ FORCE_INLINE int LZ4_decompress_generic( op += length; break; /* Necessarily EOF, due to parsing restrictions */ } - LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + //LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; /* get offset */ match = cpy - LZ4_readLE16(ip); ip+=2; @@ -1060,11 +1032,14 @@ FORCE_INLINE int LZ4_decompress_generic( if (unlikely(cpy>oend-12)) { - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */ - if (op oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ + if (op < oend-8) LZ4_wildCopy(op, match, oend-8); + match += oend-8 - op; + op = oend-8; + while (op chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } for (chunkNb=0; chunkNb Date: Fri, 28 Nov 2014 02:48:21 +0100 Subject: Fixed decompression bug --- lz4.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lz4.c b/lz4.c index b3f1b2d..2a6f038 100644 --- a/lz4.c +++ b/lz4.c @@ -1033,9 +1033,12 @@ FORCE_INLINE int LZ4_decompress_generic( if (unlikely(cpy>oend-12)) { if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ - if (op < oend-8) LZ4_wildCopy(op, match, oend-8); - match += oend-8 - op; - op = oend-8; + if (op < oend-8) + { + LZ4_wildCopy(op, match, oend-8); + match += oend-8 - op; + op = oend-8; + } while (op Date: Sat, 29 Nov 2014 16:41:28 +0100 Subject: Fixed : decompression issue on 32-bits CPU without unaligned memory access --- examples/Makefile | 2 +- lz4.c | 36 +++++++++++++++++++++++++++--------- lz4.h | 29 ++++++++--------------------- programs/lz4cli.c | 15 +-------------- 4 files changed, 37 insertions(+), 45 deletions(-) diff --git a/examples/Makefile b/examples/Makefile index 4474f59..df24ea9 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -32,7 +32,7 @@ CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) TESTFILE= Makefile diff --git a/lz4.c b/lz4.c index 2a6f038..f2a8120 100644 --- a/lz4.c +++ b/lz4.c @@ -44,10 +44,26 @@ /* * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : - * You can force the code to use unaligned memory access, should you know your CPU can handle it efficiently. - * If it effectively results in better speed (up to 50% improvement can be expected) + * By default, the source code expects the compiler to correctly optimize + * 4-bytes and 8-bytes read on architectures able to handle it efficiently. + * This is not always the case. In some circumstances (ARM notably), + * the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses. + * + * You can force the compiler to use unaligned memory access by uncommenting the line below. + * One of the below scenarios will happen : + * 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case). + * You will witness large performance improvements (+50% and up). + * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) + * The goal is to automatically detect such situations by adding your target CPU within an exception list. + * 2 - Your target CPU correctly handle unaligned access, and was already correctly optimized by compiler + * No change will be experienced. + * 3 - Your target CPU inefficiently handle unaligned access. + * You will experience a performance loss. Comment back the line. + * 4 - Your target CPU does not handle unaligned access. + * Program will crash. + * If it effectively results in better speed (case 1) * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) - * so that an automatic detection macro can be added to mainline. + * so that an automatic detection macro can be added for future versions of the library. */ /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ @@ -58,7 +74,7 @@ /* * Automated efficient unaligned memory access detection * Based on known hardware architectures - * This list will be updated thanks to Open Source community feedbacks + * This list will be updated thanks to feedbacks */ #if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ || defined(__ARM_FEATURE_UNALIGNED) \ @@ -71,7 +87,10 @@ # define LZ4_UNALIGNED_ACCESS 0 #endif -/* Define this parameter if your target system or compiler does not support hardware bit count */ +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ # define LZ4_FORCE_SW_BITCOUNT #endif @@ -88,7 +107,7 @@ #ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ +# include # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #else # ifdef __GNUC__ @@ -961,7 +980,6 @@ FORCE_INLINE int LZ4_decompress_generic( } LZ4_wildCopy(op, ip, cpy); ip += length; op = cpy; - //LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy; /* get offset */ match = cpy - LZ4_readLE16(ip); ip+=2; @@ -1018,7 +1036,7 @@ FORCE_INLINE int LZ4_decompress_generic( /* copy repeated sequence */ cpy = op + length; - if (unlikely((op-match)<(int)STEPSIZE)) + if (unlikely((op-match)<8)) { const size_t dec64 = dec64table[op-match]; op[0] = match[0]; @@ -1036,7 +1054,7 @@ FORCE_INLINE int LZ4_decompress_generic( if (op < oend-8) { LZ4_wildCopy(op, match, oend-8); - match += oend-8 - op; + match += (oend-8) - op; op = oend-8; } while (op on unix @@ -128,15 +124,6 @@ //************************************** -// Architecture Macros -//************************************** -static const int one = 1; -#define CPU_LITTLE_ENDIAN (*(char*)(&one)) -#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) -#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) - - -//************************************** // Macros //************************************** #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) @@ -462,7 +449,7 @@ int main(int argc, char** argv) } DISPLAYLEVEL(3, WELCOME_MESSAGE); - DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); + if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); // No input filename ==> use stdin if(!input_filename) { input_filename=stdinmark; } -- cgit v0.12 From 33dca250ee0afa3aec58bf3bf1a34b09bebf3fb5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 29 Nov 2014 17:12:26 +0100 Subject: minor : fixed warning under clang --- lz4hc.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lz4hc.c b/lz4hc.c index 8f6f25b..3dbdf1d 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -50,7 +50,7 @@ Memory routines /************************************** -CPU Feature Detection + CPU Feature Detection **************************************/ /* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ @@ -102,7 +102,7 @@ CPU Feature Detection /************************************** -Compiler Options + Compiler Options **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ /* "restrict" is a known keyword */ @@ -138,14 +138,14 @@ Compiler Options /************************************** -Includes + Includes **************************************/ -#include "lz4hc.h" #include "lz4.h" +#include "lz4hc.h" /************************************** -Basic Types + Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include @@ -248,9 +248,12 @@ Architecture-specific macros **************************************/ typedef struct { - U32 hashTable[HASHTABLESIZE]; + union { + U64 alignedOn8Bytes; /* force 8-bytes alignment on 32-bits systems */ + U32 hashTable[HASHTABLESIZE]; + }; U16 chainTable[MAXD]; - const BYTE* end; /* next block here to keep current prefix as prefix */ + const BYTE* end; /* next block here to continue on current prefix */ const BYTE* base; /* All index relative to this position */ const BYTE* dictBase; /* alternate base for extDict */ U32 dictLimit; /* below that point, need extDict */ -- cgit v0.12 From 6658c49a971e0363bfa17de418b4512bfe12f0c5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 29 Nov 2014 17:44:33 +0100 Subject: Improved compression speed on big endian CPU --- lz4.c | 124 +++++++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 73 insertions(+), 51 deletions(-) diff --git a/lz4.c b/lz4.c index f2a8120..fbe73bd 100644 --- a/lz4.c +++ b/lz4.c @@ -55,15 +55,15 @@ * You will witness large performance improvements (+50% and up). * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) * The goal is to automatically detect such situations by adding your target CPU within an exception list. - * 2 - Your target CPU correctly handle unaligned access, and was already correctly optimized by compiler + * 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler * No change will be experienced. * 3 - Your target CPU inefficiently handle unaligned access. * You will experience a performance loss. Comment back the line. * 4 - Your target CPU does not handle unaligned access. * Program will crash. - * If it effectively results in better speed (case 1) + * If uncommenting results in better performance (case 1) * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) - * so that an automatic detection macro can be added for future versions of the library. + * An automatic detection macro will be added to match your case within future versions of the library. */ /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ @@ -177,6 +177,7 @@ static unsigned LZ4_isLittleEndian(void) return one.c[0]; } + static U16 LZ4_readLE16(const void* memPtr) { if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) @@ -204,12 +205,10 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } -static U32 LZ4_read16(const void* memPtr) +static U16 LZ4_read16(const void* memPtr) { if (LZ4_UNALIGNED_ACCESS) - { return *(U16*)memPtr; - } else { U16 val16; @@ -221,9 +220,7 @@ static U32 LZ4_read16(const void* memPtr) static U32 LZ4_read32(const void* memPtr) { if (LZ4_UNALIGNED_ACCESS) - { return *(U32*)memPtr; - } else { U32 val32; @@ -232,36 +229,24 @@ static U32 LZ4_read32(const void* memPtr) } } - -static U32 LZ4_readLE32(const void* memPtr) -{ - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) - return *(U32*)memPtr; - { - const BYTE* p = memPtr; - U32 result = (U32)((U32)p[0] + (p[1]<<8) + (p[2]<<16) + ((U32)p[3]<<24)); - return result; - } -} - -static U64 LZ4_readLE64(const void* memPtr) +static U64 LZ4_read64(const void* memPtr) { - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + if (LZ4_UNALIGNED_ACCESS) return *(U64*)memPtr; else { - const BYTE* p = memPtr; - return (U64)((U64)p[0] + (p[1]<<8) + (p[2]<<16) + ((U64)p[3]<<24) + - (((U64)p[4])<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; } } -static size_t LZ4_readLE_ARCH(const void* p) +static size_t LZ4_read_ARCH(const void* p) { if (LZ4_64bits()) - return (size_t)LZ4_readLE64(p); + return (size_t)LZ4_read64(p); else - return (size_t)LZ4_readLE32(p); + return (size_t)LZ4_read32(p); } @@ -365,31 +350,68 @@ int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } ********************************/ static unsigned LZ4_NbCommonBytes (register size_t val) { - if (LZ4_64bits()) + if (LZ4_isLittleEndian()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } } - /* 32 bits */ + else /* Big Endian CPU */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } } @@ -439,7 +461,7 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi while (likely(pIn Date: Sat, 29 Nov 2014 20:19:39 +0100 Subject: Updated lz4hc : re-use most shared elements from lz4 (endianess / align / bus detection routines) --- lz4.c | 131 ++++++++++++--------- lz4hc.c | 312 ++++++--------------------------------------------- programs/frametest.c | 2 +- 3 files changed, 108 insertions(+), 337 deletions(-) diff --git a/lz4.c b/lz4.c index fbe73bd..9439b36 100644 --- a/lz4.c +++ b/lz4.c @@ -285,12 +285,8 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) /************************************** - Constants + Common Constants **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) - #define MINMATCH 4 #define COPYLENGTH 8 @@ -298,13 +294,10 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) #define MFLIMIT (COPYLENGTH+MINMATCH) static const int LZ4_minLength = (MFLIMIT+1); -#define KB *(1U<<10) -#define MB *(1U<<20) +#define KB *(1 <<10) +#define MB *(1 <<20) #define GB *(1U<<30) -#define LZ4_64KLIMIT ((64 KB) + (MFLIMIT-1)) -#define SKIPSTRENGTH 6 /* Increasing this value will make the compression run slower on incompressible data */ - #define MAXD_LOG 16 #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) @@ -315,38 +308,13 @@ static const int LZ4_minLength = (MFLIMIT+1); /************************************** - Structures and local types -**************************************/ -typedef struct { - U32 hashTable[HASH_SIZE_U32]; - U32 currentOffset; - U32 initCheck; - const BYTE* dictionary; - const BYTE* bufferStart; - U32 dictSize; -} LZ4_stream_t_internal; - -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - - -/************************************** - Utils + Common Utils **************************************/ #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } - /******************************** - Compression functions + Common functions ********************************/ static unsigned LZ4_NbCommonBytes (register size_t val) { @@ -415,6 +383,73 @@ static unsigned LZ4_NbCommonBytes (register size_t val) } } +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn /* calloc, free */ -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/************************************** - CPU Feature Detection -**************************************/ -/* 32 or 64 bits ? */ -#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__64BIT__) || defined(__mips64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \ - || defined(__s390x__) ) /* Detects 64 bits mode */ -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 -#endif - -/* -* Little Endian or Big Endian ? -* Overwrite the #define below if you know your architecture endianess -*/ -#include /* Apparently required to detect endianess */ -#if defined (__GLIBC__) -# include -# if (__BYTE_ORDER == __BIG_ENDIAN) -# define LZ4_BIG_ENDIAN 1 -# endif -#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) -# define LZ4_BIG_ENDIAN 1 -#elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) -# define LZ4_BIG_ENDIAN 1 -#else -/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ -#endif - -/* -* Unaligned memory access is automatically enabled for "common" CPU, such as x86. -* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected -* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance -*/ -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 -#endif - -/* Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/************************************** - Compiler Options -**************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# if LZ4_ARCH64 /* 64-bits */ -# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ -# else /* 32-bits */ -# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ -# endif -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define lz4_bswap16(x) _byteswap_ushort(x) -#else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - - -/************************************** Includes **************************************/ #include "lz4.h" @@ -145,106 +47,40 @@ Memory routines /************************************** - Basic Types + Local Compiler Options **************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" #endif -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif +#if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" #endif -typedef struct _U16_S { U16 v; } _PACKED U16_S; -typedef struct _U32_S { U32 v; } _PACKED U32_S; -typedef struct _U64_S { U64 v; } _PACKED U64_S; - -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) +/************************************** + Common LZ4 definition +**************************************/ +#define LZ4_COMMONDEFS_ONLY +#include "lz4.c" /************************************** -Constants + Local Constants **************************************/ -#define MINMATCH 4 - #define DICTIONARY_LOGSIZE 16 #define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) #define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] #define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) -static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(A32(ptr)); } - -/************************************** -Private functions -**************************************/ -#if LZ4_ARCH64 +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } -FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; -# endif -#endif -} - -#else - -FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanReverse( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif -#endif -} - -#endif +/************************************** + HC Compression +**************************************/ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); @@ -397,24 +166,6 @@ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newB } -static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit) -{ - const BYTE* const p1Start = p1; - - while (p1 <= p1Limit - STEPSIZE) - { - size_t diff = AARCH(p2) ^ AARCH(p1); - if (!diff) { p1+=STEPSIZE; p2+=STEPSIZE; continue; } - p1 += LZ4_NbCommonBytes(diff); - return (p1 - p1Start); - } - if (LZ4_ARCH64) if ((p1<(p1Limit-3)) && (A32(p2) == A32(p1))) { p1+=4; p2+=4; } - if ((p1<(p1Limit-1)) && (A16(p2) == A16(p1))) { p1+=2; p2+=2; } - if ((p1 ml) { ml = mlt; *matchpos = match; } } } else { match = dictBase + matchIndex; - if (A32(match) == A32(ip)) + if (LZ4_read32(match) == LZ4_read32(ip)) { size_t mlt; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iLimit) vLimit = iLimit; - mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iLimit)) - mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iLimit); + mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos } } @@ -502,11 +253,11 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( { match = base + matchIndex; if (*(iLowLimit + longest) == *(match - delta + longest)) - if (A32(match) == A32(ip)) + if (LZ4_read32(match) == LZ4_read32(ip)) { const BYTE* startt = ip; const BYTE* tmpMatch = match; - const BYTE* const matchEnd = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, iHighLimit); + const BYTE* const matchEnd = ip + MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} @@ -521,15 +272,15 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( else { match = dictBase + matchIndex; - if (A32(match) == A32(ip)) + if (LZ4_read32(match) == LZ4_read32(ip)) { size_t mlt; int back=0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4HC_CommonLength(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4HC_CommonLength(ip+mlt, base+dictLimit, iHighLimit); + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; mlt -= back; if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } @@ -568,10 +319,11 @@ FORCE_INLINE int LZ4HC_encodeSequence ( else *token = (BYTE)(length< Date: Sun, 30 Nov 2014 12:58:00 +0100 Subject: Fixed : some minor Visual warnings --- lz4.c | 2 +- lz4hc.c | 4 ++++ programs/Makefile | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lz4.c b/lz4.c index 9439b36..fb84955 100644 --- a/lz4.c +++ b/lz4.c @@ -369,7 +369,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; - _BitScanReverse( &r, val ); + _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clz(val) >> 3); diff --git a/lz4hc.c b/lz4hc.c index a798cab..89f0db0 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -57,6 +57,10 @@ You can contact the author at : # pragma clang diagnostic ignored "-Wunused-function" #endif +#if defined(_MSC_VER) /* Visual Studio */ +# pragma warning(disable : 4201) /* disable: C4201: unnamed struct/union*/ +#endif + /************************************** Common LZ4 definition diff --git a/programs/Makefile b/programs/Makefile index fcfb32c..8a3ed95 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -193,7 +193,7 @@ test-mem: lz4 datagen fuzzer frametest ./datagen -g16MB > tmp valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null ./datagen -g256MB > tmp - valgrind --leak-check=yes ./lz4 -B4D -f tmp /dev/null + valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null rm tmp valgrind --leak-check=yes ./fuzzer -i50 -t0 valgrind --leak-check=yes ./frametest -i100 -- cgit v0.12 From 6de52c2a8c40b7381e364ad87af9e80a60d95229 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Nov 2014 17:59:31 +0100 Subject: LZ4IO : modified to use lz4frame instead --- lz4frame.h | 2 +- programs/Makefile | 19 +- programs/lz4cli.c | 3 +- programs/lz4io.c | 595 ++++++++++++++---------------------------------------- programs/lz4io.h | 4 +- 5 files changed, 167 insertions(+), 456 deletions(-) diff --git a/lz4frame.h b/lz4frame.h index d31203d..a4f5e36 100644 --- a/lz4frame.h +++ b/lz4frame.h @@ -201,7 +201,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB typedef void* LZ4F_decompressionContext_t; typedef struct { - unsigned stableDst; /* unused for the time being, must be 0 */ + unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ unsigned reserved[3]; } LZ4F_decompressOptions_t; diff --git a/programs/Makefile b/programs/Makefile index 8a3ed95..9001e2c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -65,13 +65,13 @@ default: lz4 lz4c all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen -lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c +lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c $(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c +lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c $(CC) $(FLAGS) $^ -o $@$(EXT) -lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c +lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c @@ -80,16 +80,16 @@ fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xx fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c +fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c +frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c +frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) datagen : datagen.c @@ -156,9 +156,10 @@ test-lz4: lz4 datagen @rm *.test @echo frame concatenation test completed # test frame concatenation with null-length frame - + test-lz4c: lz4c datagen + ./datagen -g256MB | ./lz4c -l -v -B4D | ./lz4c -vdq > $(VOID) test-lz4c32: lz4 lz4c32 lz4 datagen ./datagen -g16KB | ./lz4c32 -9 | ./lz4c32 -vdq > $(VOID) @@ -195,8 +196,8 @@ test-mem: lz4 datagen fuzzer frametest ./datagen -g256MB > tmp valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null rm tmp - valgrind --leak-check=yes ./fuzzer -i50 -t0 - valgrind --leak-check=yes ./frametest -i100 + valgrind --leak-check=yes ./fuzzer -i64 -t0 + valgrind --leak-check=yes ./frametest -i256 test-mem32: lz4c32 datagen # unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system... diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 6e52ec6..8a8fc27 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -48,7 +48,6 @@ # pragma warning(disable : 4127) // disable: C4127: conditional expression is constant #endif -#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix #define _POSIX_SOURCE 1 // for fileno() within on unix @@ -400,7 +399,7 @@ int main(int argc, char** argv) argument++; break; } - case 'D': LZ4IO_setBlockMode(chainedBlocks); argument++; break; + case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break; case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; default : exitBlockProperties=1; } diff --git a/programs/lz4io.c b/programs/lz4io.c index afaa59f..a9c3c97 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -38,6 +38,12 @@ # pragma warning(disable : 4127) // disable: C4127: conditional expression is constant #endif +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + #define _LARGE_FILES // Large file support on 32-bits AIX #define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix #define _POSIX_SOURCE 1 // for fileno() within on unix @@ -47,13 +53,13 @@ // Includes //**************************** #include // fprintf, fopen, fread, _fileno, stdin, stdout -#include // malloc +#include // malloc, free #include // strcmp, strlen #include // clock #include "lz4io.h" -#include "lz4.h" -#include "lz4hc.h" -#include "xxhash.h" +#include "lz4.h" // still required for legacy format +#include "lz4hc.h" // still required for legacy format +#include "lz4frame.h" //**************************** @@ -74,31 +80,11 @@ #endif -//************************************** -// Compiler-specific functions -//************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if defined(_MSC_VER) // Visual Studio -# define swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define swap32 __builtin_bswap32 -#else - static unsigned int swap32(unsigned int x) - { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); - } -#endif - - //**************************** // Constants //**************************** -#define KB *(1U<<10) -#define MB *(1U<<20) +#define KB *(1 <<10) +#define MB *(1 <<20) #define GB *(1U<<30) #define _1BIT 0x01 @@ -123,19 +109,16 @@ //************************************** -// Architecture Macros -//************************************** -static const int one = 1; -#define CPU_LITTLE_ENDIAN (*(char*)(&one)) -#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) -#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) - - -//************************************** // Macros //************************************** #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ + if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stdout); } } +static const unsigned refreshRate = 150; +static clock_t g_time = 0; //************************************** @@ -196,9 +179,9 @@ int LZ4IO_setBlockSizeID(int bsid) return blockSizeTable[globalBlockSizeId-minBlockSizeID]; } -int LZ4IO_setBlockMode(blockMode_t blockMode) +int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) { - blockIndependence = (blockMode == independentBlocks); + blockIndependence = (blockMode == LZ4IO_blockIndependent); return blockIndependence; } @@ -223,6 +206,13 @@ int LZ4IO_setNotificationLevel(int level) return displayLevel; } +static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious) +{ +#define CLOCKS_PER_MSEC (CLOCKS_PER_SEC/1000) + clock_t nCurrent = clock(); + unsigned nSpan = (unsigned)((nCurrent - nPrevious) / CLOCKS_PER_MSEC); + return nSpan; +} /* ************************************************************************ */ @@ -230,7 +220,6 @@ int LZ4IO_setNotificationLevel(int level) /* ************************************************************************ */ static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } @@ -282,8 +271,25 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf } -// LZ4IO_compressFilename_Legacy : This function is intentionally "hidden" (not published in .h) -// It generates compressed streams using the old 'legacy' format + + +/*************************************** + * Legacy Compression + * *************************************/ + +/* unoptimized version; solves endianess & alignment issues */ +static void LZ4IO_writeLE32 (void* p, unsigned value32) +{ + unsigned char* dstPtr = p; + dstPtr[0] = (unsigned char)value32; + dstPtr[1] = (unsigned char)(value32 >> 8); + dstPtr[2] = (unsigned char)(value32 >> 16); + dstPtr[3] = (unsigned char)(value32 >> 24); +} + +/* LZ4IO_compressFilename_Legacy : + * This function is intentionally "hidden" (not published in .h) + * It generates compressed streams using the old 'legacy' format */ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel) { int (*compressionFunction)(const char*, char*, int); @@ -298,8 +304,9 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i // Init - if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; start = clock(); + if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + get_fileHandle(input_filename, output_filename, &finput, &foutput); if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; @@ -309,7 +316,7 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); + LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER); sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); @@ -321,15 +328,14 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); if( inSize<=0 ) break; filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); // Compress Block outSize = compressionFunction(in_buff, out_buff+4, inSize); compressedfilesize += outSize+4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); // Write Block - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + LZ4IO_writeLE32(out_buff, outSize); sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); } @@ -354,320 +360,93 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i } -static void* LZ4IO_LZ4_createStream (const char* inputBuffer) -{ - (void)inputBuffer; - return calloc(8, LZ4_STREAMSIZE_U64); -} - -static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) -{ - (void)compressionLevel; - return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize); -} - -static int LZ4IO_LZ4_saveDict (void* LZ4_stream, char* safeBuffer, int dictSize) -{ - return LZ4_saveDict ((LZ4_stream_t*) LZ4_stream, safeBuffer, dictSize); -} - -static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size) -{ - (void)size; (void)buffer; - LZ4_slideInputBufferHC (ctx); - return 1; -} - - -static int LZ4IO_free (void* ptr) -{ - free(ptr); - return 0; -} - -static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) -{ - void* (*initFunction) (const char*); - int (*compressionFunction)(void*, const char*, char*, int, int, int); - int (*nextBlockFunction) (void*, char*, int); - int (*freeFunction) (void*); - void* ctx; - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - unsigned int checkbits; - char* in_buff, *in_blockStart; - char* out_buff; - FILE* finput; - FILE* foutput; - clock_t start, end; - unsigned int blockSize, inputBufferSize; - size_t sizeCheck, header_size; - XXH32_state_t streamCRC; - - // Init - start = clock(); - if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - - if (compressionlevel<3) - { - initFunction = LZ4IO_LZ4_createStream; - compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue; - nextBlockFunction = LZ4IO_LZ4_saveDict; - freeFunction = LZ4IO_free; - } - else - { - initFunction = LZ4_createHC; - compressionFunction = LZ4_compressHC2_limitedOutput_continue; - nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC; - freeFunction = LZ4IO_free; - } - - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId); - - // Allocate Memory - inputBufferSize = 64 KB + blockSize; - in_buff = (char*)malloc(inputBufferSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); - in_blockStart = in_buff + 64 KB; - if (compressionlevel>=3) in_blockStart = in_buff; - if (streamChecksum) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); - ctx = initFunction(in_buff); - - // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') - *(out_buff+4) |= (blockIndependence & _1BIT) << 5; - *(out_buff+4) |= (blockChecksum & _1BIT) << 4; - *(out_buff+4) |= (streamChecksum & _1BIT) << 2; - *(out_buff+5) = (char)((globalBlockSizeId & _3BITS) << 4); - checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(out_buff+6) = (unsigned char) checkbits; - header_size = 7; - sizeCheck = fwrite(out_buff, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; - - // Main Loop - while (1) - { - unsigned int outSize; - unsigned int inSize; - - // Read Block - inSize = (unsigned int) fread(in_blockStart, (size_t)1, (size_t)blockSize, finput); - if( inSize==0 ) break; // No more input : end of compression - filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(&streamCRC, in_blockStart, inSize); - - // Compress Block - outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100); - - // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - } - else // Copy Original - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_blockStart, 1, inSize, foutput); - if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_blockStart, inSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } - { - size_t sizeToMove = 64 KB; - if (inSize < 64 KB) sizeToMove = inSize; - nextBlockFunction(ctx, in_blockStart - sizeToMove, (int)sizeToMove); - if (compressionlevel>=3) in_blockStart = in_buff + 64 KB; - } - } - - // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(&streamCRC); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } - - // Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - freeFunction(ctx); - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -static int LZ4_compress_limitedOutput_local(const char* src, char* dst, int size, int maxOut, int clevel) -{ (void)clevel; return LZ4_compress_limitedOutput(src, dst, size, maxOut); } +/*********************************************** + * Compression using Frame format + * ********************************************/ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel) { - int (*compressionFunction)(const char*, char*, int, int, int); unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; - unsigned int checkbits; char* in_buff; char* out_buff; - char* headerBuffer; FILE* finput; FILE* foutput; clock_t start, end; int blockSize; - size_t sizeCheck, header_size, readSize; - XXH32_state_t streamCRC; + size_t sizeCheck, headerSize, readSize, outBuffSize; + LZ4F_compressionContext_t ctx; + LZ4F_errorCode_t errorCode; + LZ4F_preferences_t prefs = {0}; - // Branch out - if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionLevel); // Init start = clock(); if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3; - if (compressionLevel <= 3) compressionFunction = LZ4_compress_limitedOutput_local; - else { compressionFunction = LZ4_compressHC2_limitedOutput; } + errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); get_fileHandle(input_filename, output_filename, &finput, &foutput); blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId); + // Set compression parameters + prefs.autoFlush = 1; + prefs.compressionLevel = compressionLevel; + prefs.frameInfo.blockMode = blockIndependence; + prefs.frameInfo.blockSizeID = globalBlockSizeId; + prefs.frameInfo.contentChecksumFlag = streamChecksum; + // Allocate Memory in_buff = (char*)malloc(blockSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); - if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); - if (streamChecksum) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); + outBuffSize = LZ4F_compressBound(blockSize, &prefs); + out_buff = (char*)malloc(outBuffSize); + if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); // Write Archive Header - *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') - *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; - *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; - *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; - *(headerBuffer+5) = (char)((globalBlockSizeId & _3BITS) << 4); - checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(headerBuffer+6) = (unsigned char) checkbits; - header_size = 7; - - // Write header - sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; + headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs); + if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + sizeCheck = fwrite(out_buff, 1, headerSize, foutput); + if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); + compressedfilesize += headerSize; // read first block readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + filesize += readSize; // Main Loop while (readSize>0) { - unsigned int outSize; - - filesize += readSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(&streamCRC, in_buff, (int)readSize); + size_t outSize; // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1, compressionLevel); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100); + outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL); + if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + compressedfilesize += outSize; + DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - } - else // Copy Original Uncompressed - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_buff, 1, readSize, foutput); - if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } + sizeCheck = fwrite(out_buff, 1, outSize, foutput); + if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); // Read next block readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + filesize += readSize; } // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(&streamCRC); - *(unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } + headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL); + if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + + sizeCheck = fwrite(out_buff, 1, headerSize, foutput); + if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += headerSize; // Close & Free free(in_buff); free(out_buff); - free(headerBuffer); fclose(finput); fclose(foutput); + errorCode = LZ4F_freeCompressionContext(ctx); + if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); // Final Status end = clock(); @@ -687,6 +466,16 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp /* ********************** LZ4 File / Stream decoding ******************* */ /* ********************************************************************* */ +static unsigned LZ4IO_readLE32 (const void* s) +{ + const unsigned char* srcPtr = s; + unsigned value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += (srcPtr[3]<<24); + return value32; +} + static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; @@ -694,7 +483,6 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) char* out_buff; unsigned int blockSize; - // Allocate Memory in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); out_buff = (char*)malloc(LEGACY_BLOCKSIZE); @@ -707,9 +495,9 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) size_t sizeCheck; // Block Size - sizeCheck = fread(&blockSize, 1, 4, finput); + sizeCheck = fread(in_buff, 1, 4, finput); if (sizeCheck==0) break; // Nothing to read : file read is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian + blockSize = LZ4IO_readLE32(in_buff); // Convert to Little Endian if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { // Cannot read next block : maybe new stream ? fseek(finput, -4, SEEK_CUR); @@ -740,142 +528,64 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; - char* in_buff; - char* out_buff, *out_start, *out_end; - unsigned char descriptor[LZ4S_MAXHEADERSIZE]; - size_t nbReadBytes; - int decodedBytes=0; - unsigned int maxBlockSize; - size_t sizeCheck; - int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; - XXH32_state_t streamCRC; - int (*decompressionFunction)(LZ4_streamDecode_t* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue; - LZ4_streamDecode_t ctx; + char* inBuff; + char* outBuff; +# define HEADERMAX 20 + char headerBuff[HEADERMAX]; + size_t sizeCheck, nextToRead, outBuffSize, inBuffSize; + LZ4F_decompressionContext_t ctx; + LZ4F_errorCode_t errorCode; + LZ4F_frameInfo_t frameInfo; // init - memset(&ctx, 0, sizeof(ctx)); + errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode)); + LZ4IO_writeLE32(headerBuff, LZ4S_MAGICNUMBER); /* regenerated here, as it was already read from finput */ // Decode stream descriptor - nbReadBytes = fread(descriptor, 1, 3, finput); - if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); - { - int version = (descriptor[0] >> 6) & _2BITS; - int streamSize = (descriptor[0] >> 3) & _1BIT; - int reserved1 = (descriptor[0] >> 1) & _1BIT; - int dictionary = (descriptor[0] >> 0) & _1BIT; - - int reserved2 = (descriptor[1] >> 7) & _1BIT; - int blockSizeId = (descriptor[1] >> 4) & _3BITS; - int reserved3 = (descriptor[1] >> 0) & _4BITS; - int checkBits = (descriptor[2] >> 0) & _8BITS; - int checkBits_xxh32; - - blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; - blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; - streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; - - if (version != 1) EXM_THROW(62, "Wrong version number"); - if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); - if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); - if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); - if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); - if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); - // Checkbits verification - descriptor[1] &= 0xF0; - checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); - checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); - if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); - } + outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE; + nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); + if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead)); + if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX); + sizeCheck = fread(headerBuff, 1, nextToRead, finput); + if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error "); + nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL); + errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize); + if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode)); // Allocate Memory - { - size_t outBuffSize = maxBlockSize + 64 KB; - if (outBuffSize < MIN_STREAM_BUFSIZE) outBuffSize = MIN_STREAM_BUFSIZE; - in_buff = (char*)malloc(maxBlockSize); - out_buff = (char*)malloc(outBuffSize); - out_start = out_buff; - out_end = out_start + outBuffSize; - if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); - if (streamChecksumFlag) XXH32_reset(&streamCRC, LZ4S_CHECKSUM_SEED); - } + outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID); + inBuffSize = outBuffSize + 4; + inBuff = (char*)malloc(inBuffSize); + outBuff = (char*)malloc(outBuffSize); + if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory"); // Main Loop - while (1) + while (nextToRead != 0) { - unsigned int blockSize, uncompressedFlag; - - // Block Size - nbReadBytes = fread(&blockSize, 1, 4, finput); - if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); - if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian - uncompressedFlag = blockSize >> 31; - blockSize &= 0x7FFFFFFF; - if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); + size_t decodedBytes = outBuffSize; // Read Block - nbReadBytes = fread(in_buff, 1, blockSize, finput); - if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); - - // Check Block - if (blockChecksumFlag) - { - unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); - } - - if (uncompressedFlag) - { - // Write uncompressed Block - sizeCheck = fwrite(in_buff, 1, blockSize, foutput); - if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); - filesize += blockSize; - if (streamChecksumFlag) XXH32_update(&streamCRC, in_buff, blockSize); - if (!blockIndependenceFlag) - { - // handle dictionary for streaming - memcpy(in_buff + blockSize - 64 KB, out_buff, 64 KB); - LZ4_setStreamDecode(&ctx, out_buff, 64 KB); - out_start = out_buff + 64 KB; - } - } - else - { - // Decode Block - if (out_start + maxBlockSize > out_end) out_start = out_buff; - decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize); - if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); - filesize += decodedBytes; - if (streamChecksumFlag) XXH32_update(&streamCRC, out_start, decodedBytes); - - // Write Block - sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); - if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); - out_start += decodedBytes; - } + sizeCheck = fread(inBuff, 1, nextToRead, finput); + if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error "); - } + // Decode Block + errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL); + if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode)); + if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error"); + nextToRead = errorCode; + filesize += decodedBytes; - // Stream Checksum - if (streamChecksumFlag) - { - unsigned int checksum = XXH32_digest(&streamCRC); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(79, "Error : invalid stream checksum detected"); + // Write Block + sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); + if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n"); } // Free - free(in_buff); - free(out_buff); + free(inBuff); + free(outBuff); + errorCode = LZ4F_freeDecompressionContext(ctx); + if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); return filesize; } @@ -884,15 +594,16 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) #define ENDOFSTREAM ((unsigned long long)-1) static unsigned long long selectDecoder( FILE* finput, FILE* foutput) { - unsigned int magicNumber, size; + unsigned char U32store[MAGICNUMBER_SIZE]; + unsigned magicNumber, size; int errorNb; size_t nbReadBytes; // Check Archive Header - nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); + nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); if (nbReadBytes==0) return ENDOFSTREAM; // EOF - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); - magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LZ4IO_readLE32(U32store); // Convert to Little Endian format if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers switch(magicNumber) @@ -904,9 +615,9 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) return decodeLegacyStream(finput, foutput); case LZ4S_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(&size, 1, 4, finput); + nbReadBytes = fread(U32store, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format + size = LZ4IO_readLE32(U32store); // Convert to Little Endian format errorNb = fseek(finput, size, SEEK_CUR); if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); return selectDecoder(finput, foutput); diff --git a/programs/lz4io.h b/programs/lz4io.h index 9c3b217..7869a43 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -64,8 +64,8 @@ int LZ4IO_setOverwrite(int yes); int LZ4IO_setBlockSizeID(int blockSizeID); /* Default setting : independent blocks */ -typedef enum { chainedBlocks, independentBlocks } blockMode_t; -int LZ4IO_setBlockMode(blockMode_t blockMode); +typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; +int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); /* Default setting : no checksum */ int LZ4IO_setBlockChecksumMode(int xxhash); -- cgit v0.12 From 80030c16fff7a8a5dae18cf2b1ab5da508137847 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Nov 2014 18:04:32 +0100 Subject: Update cmakelist to support new lz4io version --- cmake_unofficial/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt index 1d70590..38d10b1 100644 --- a/cmake_unofficial/CMakeLists.txt +++ b/cmake_unofficial/CMakeLists.txt @@ -2,7 +2,7 @@ PROJECT(LZ4 C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library") set(CPACK_PACKAGE_VERSION_MAJOR 1) set(CPACK_PACKAGE_VERSION_MINOR 4) -set(CPACK_PACKAGE_VERSION_PATCH r124) +set(CPACK_PACKAGE_VERSION_PATCH r125) set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") include(CPack) @@ -26,7 +26,7 @@ endif() set(LZ4_DIR ../) set(PRG_DIR ../programs/) set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h) -set(LZ4_SRCS ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) +set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) if(BUILD_TOOLS AND NOT BUILD_LIBS) set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) -- cgit v0.12 From d008c87151abf8c36a9f98d28461bf6f3dfdc6ae Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Nov 2014 23:32:12 +0100 Subject: New directory structure : library source files into /lib directory --- .travis.yml | 2 +- LICENSE | 24 - Makefile | 77 +-- NEWS | 2 + cmake_unofficial/CMakeLists.txt | 2 +- examples/Makefile | 4 +- lib/LICENSE | 24 + lib/Makefile | 118 ++++ lib/liblz4.pc | 14 + lib/liblz4.pc.in | 14 + lib/liblz4.so | 1 + lib/liblz4.so.1 | 1 + lib/liblz4.so.1.4.1 | Bin 0 -> 68571 bytes lib/lz4.c | 1365 +++++++++++++++++++++++++++++++++++++++ lib/lz4.h | 310 +++++++++ lib/lz4.o | Bin 0 -> 47472 bytes lib/lz4frame.c | 1330 ++++++++++++++++++++++++++++++++++++++ lib/lz4frame.h | 261 ++++++++ lib/lz4hc.c | 768 ++++++++++++++++++++++ lib/lz4hc.h | 223 +++++++ lib/lz4hc.o | Bin 0 -> 15888 bytes lib/xxhash.c | 928 ++++++++++++++++++++++++++ lib/xxhash.h | 156 +++++ lz4.c | 1365 --------------------------------------- lz4.h | 310 --------- lz4frame.c | 1330 -------------------------------------- lz4frame.h | 261 -------- lz4hc.c | 768 ---------------------- lz4hc.h | 223 ------- programs/Makefile | 4 +- programs/lz4cli.c | 6 +- xxhash.c | 928 -------------------------- xxhash.h | 156 ----- 33 files changed, 5538 insertions(+), 5437 deletions(-) delete mode 100644 LICENSE create mode 100644 lib/LICENSE create mode 100644 lib/Makefile create mode 100644 lib/liblz4.pc create mode 100644 lib/liblz4.pc.in create mode 120000 lib/liblz4.so create mode 120000 lib/liblz4.so.1 create mode 100755 lib/liblz4.so.1.4.1 create mode 100644 lib/lz4.c create mode 100644 lib/lz4.h create mode 100644 lib/lz4.o create mode 100644 lib/lz4frame.c create mode 100644 lib/lz4frame.h create mode 100644 lib/lz4hc.c create mode 100644 lib/lz4hc.h create mode 100644 lib/lz4hc.o create mode 100644 lib/xxhash.c create mode 100644 lib/xxhash.h delete mode 100644 lz4.c delete mode 100644 lz4.h delete mode 100644 lz4frame.c delete mode 100644 lz4frame.h delete mode 100644 lz4hc.c delete mode 100644 lz4hc.h delete mode 100644 xxhash.c delete mode 100644 xxhash.h diff --git a/.travis.yml b/.travis.yml index 67bde9b..a8866a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - sudo apt-get install -qq valgrind env: - - LZ4_TRAVIS_CI_ENV=liblz4 + - LZ4_TRAVIS_CI_ENV=install - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake - LZ4_TRAVIS_CI_ENV=dist diff --git a/LICENSE b/LICENSE deleted file mode 100644 index b566df3..0000000 --- a/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -LZ4 Library -Copyright (c) 2011-2014, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/Makefile b/Makefile index c2bbd57..60f8ce1 100644 --- a/Makefile +++ b/Makefile @@ -33,10 +33,6 @@ # Version numbers VERSION=125 export RELEASE=r$(VERSION) -LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` -LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` -LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` -LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= PREFIX ?= /usr @@ -47,27 +43,13 @@ CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prot LIBDIR?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include PRGDIR = programs +LZ4DIR = lib DISTRIBNAME=lz4-$(RELEASE).tar.gz - -# OS X linker doesn't support -soname, and use different extension -# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(shell uname), Darwin) - SHARED_EXT = dylib - SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) - SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) -else - SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT = so - SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) -endif - -TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ - lz4frame.c lz4frame.h xxhash.c xxhash.h \ - liblz4.pc.in Makefile \ - lz4_format_description.txt NEWS LICENSE README.md \ +TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ + $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4frame.h $(LZ4DIR)/xxhash.c $(LZ4DIR)/xxhash.h \ + $(LZ4DIR)/liblz4.pc.in $(LZ4DIR)/Makefile $(LZ4DIR)/LICENSE \ + Makefile lz4_format_description.txt NEWS README.md \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \ @@ -90,27 +72,18 @@ TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV) endif -default: liblz4 +default: lz4programs @cd $(PRGDIR); $(MAKE) -e -all: liblz4 lz4programs +all: lz4programs -lz4programs: lz4.c lz4hc.c +lz4programs: @cd $(PRGDIR); $(MAKE) -e all -liblz4: lz4.c lz4hc.c - @echo compiling static library - @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ - @$(AR) rcs liblz4.a lz4.o lz4hc.o - @echo compiling dynamic library $(LIBVER) - @$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) - @echo creating versioned links - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) - @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) - clean: - @rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* $(DISTRIBNAME) *.sha1 liblz4.pc + @rm -f $(DISTRIBNAME) *.sha1 @cd $(PRGDIR); $(MAKE) clean + @cd $(LZ4DIR); $(MAKE) clean @cd examples; $(MAKE) clean @echo Cleaning completed @@ -119,38 +92,16 @@ clean: #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) -liblz4.pc: liblz4.pc.in Makefile - @echo creating pkgconfig - @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@VERSION@|$(VERSION)|' \ - $< >$@ - -install: liblz4 liblz4.pc - @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) - @cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) - @cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) - @cp -a liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ - @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a - @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h - @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @echo lz4 static and shared library installed +install: + @cd $(LZ4DIR); $(MAKE) -e install @cd $(PRGDIR); $(MAKE) -e install uninstall: - rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) - rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) - rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc - [ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) - [ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a - [ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h - [ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h - @echo lz4 libraries successfully uninstalled + @cd $(LZ4DIR); $(MAKE) uninstall @cd $(PRGDIR); $(MAKE) uninstall dist: clean + @install -dD -m 700 lz4-$(RELEASE)/lib/ @install -dD -m 700 lz4-$(RELEASE)/programs/ @install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/ @install -dD -m 700 lz4-$(RELEASE)/images/ diff --git a/NEWS b/NEWS index 763fa5a..f49535d 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ r125: Changed : endian and alignment code +Changed : directory structure : new "lib" directory +Updated : lz4io, now uses lz4frame Fixed : some alignment warnings under clang r124: diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt index 38d10b1..cacaca1 100644 --- a/cmake_unofficial/CMakeLists.txt +++ b/cmake_unofficial/CMakeLists.txt @@ -23,7 +23,7 @@ if(UNIX AND BUILD_LIBS) endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") endif() -set(LZ4_DIR ../) +set(LZ4_DIR ../lib/) set(PRG_DIR ../programs/) set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h) set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) diff --git a/examples/Makefile b/examples/Makefile index df24ea9..0c4cf13 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -33,10 +33,10 @@ CC := $(CC) CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug -FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) TESTFILE= Makefile -LZ4DIR=.. +LZ4DIR = ../lib # Minimize test target for Travis CI's Build Matrix diff --git a/lib/LICENSE b/lib/LICENSE new file mode 100644 index 0000000..b566df3 --- /dev/null +++ b/lib/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2011-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/lib/Makefile b/lib/Makefile new file mode 100644 index 0000000..f2d585f --- /dev/null +++ b/lib/Makefile @@ -0,0 +1,118 @@ +# ################################################################ +# LZ4 library - Makefile +# Copyright (C) Yann Collet 2011-2014 +# All rights reserved. +# +# BSD license +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, this +# list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# You can contact the author at : +# - LZ4 source repository : http://code.google.com/p/lz4/ +# - LZ4 source mirror : https://github.com/Cyan4973/lz4 +# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c +# ################################################################ + +# Version numbers +RELEASE=r125 +LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` +LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` +LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` +LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) + +DESTDIR?= +PREFIX ?= /usr +CC := $(CC) +CFLAGS ?= -O3 +CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" + +LIBDIR?= $(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include + + +# OS X linker doesn't support -soname, and use different extension +# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html +ifeq ($(shell uname), Darwin) + SHARED_EXT = dylib + SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) + SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) + SONAME_FLAGS = -install_name $(PREFIX)/lib/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) +else + SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT = so + SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) +endif + +default: liblz4 + @cd $(PRGDIR); $(MAKE) -e + +all: liblz4 + +liblz4: lz4.c lz4hc.c + @echo compiling static library + @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ + @$(AR) rcs liblz4.a lz4.o lz4hc.o + @echo compiling dynamic library $(LIBVER) + @$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) + @echo creating versioned links + @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) + @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) + +clean: + @rm -f core *.o *.a *.$(SHARED_EXT) *.$(SHARED_EXT).* liblz4.pc + @echo Cleaning library completed + + +#------------------------------------------------------------------------ +#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) + +liblz4.pc: liblz4.pc.in Makefile + @echo creating pkgconfig + @sed -e 's|@PREFIX@|$(PREFIX)|' \ + -e 's|@LIBDIR@|$(LIBDIR)|' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@VERSION@|$(VERSION)|' \ + $< >$@ + +install: liblz4 liblz4.pc + @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ + @install -m 755 liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) + @cp -a liblz4.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) + @cp -a liblz4.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) + @cp -a liblz4.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ + @install -m 644 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @install -m 644 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @install -m 644 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @echo lz4 static and shared library installed + +uninstall: + @rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT) + @rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR) + @rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/liblz4.pc + @[ -x $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_VER) + @[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a + @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @echo lz4 libraries successfully uninstalled + +endif diff --git a/lib/liblz4.pc b/lib/liblz4.pc new file mode 100644 index 0000000..39983ba --- /dev/null +++ b/lib/liblz4.pc @@ -0,0 +1,14 @@ +# LZ4 - Fast LZ compression algorithm +# Copyright (C) 2011-2014, Yann Collet. +# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +prefix=/usr +libdir=/usr/lib +includedir=/usr/include + +Name: lz4 +Description: fast lossless compression algorithm library +URL: http://code.google.com/p/lz4/ +Version: +Libs: -L/usr/lib -llz4 +Cflags: -I/usr/include diff --git a/lib/liblz4.pc.in b/lib/liblz4.pc.in new file mode 100644 index 0000000..0d05152 --- /dev/null +++ b/lib/liblz4.pc.in @@ -0,0 +1,14 @@ +# LZ4 - Fast LZ compression algorithm +# Copyright (C) 2011-2014, Yann Collet. +# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +prefix=@PREFIX@ +libdir=@LIBDIR@ +includedir=@INCLUDEDIR@ + +Name: lz4 +Description: fast lossless compression algorithm library +URL: http://code.google.com/p/lz4/ +Version: @VERSION@ +Libs: -L@LIBDIR@ -llz4 +Cflags: -I@INCLUDEDIR@ diff --git a/lib/liblz4.so b/lib/liblz4.so new file mode 120000 index 0000000..3c57595 --- /dev/null +++ b/lib/liblz4.so @@ -0,0 +1 @@ +liblz4.so.1.4.1 \ No newline at end of file diff --git a/lib/liblz4.so.1 b/lib/liblz4.so.1 new file mode 120000 index 0000000..3c57595 --- /dev/null +++ b/lib/liblz4.so.1 @@ -0,0 +1 @@ +liblz4.so.1.4.1 \ No newline at end of file diff --git a/lib/liblz4.so.1.4.1 b/lib/liblz4.so.1.4.1 new file mode 100755 index 0000000..6ff6fcf Binary files /dev/null and b/lib/liblz4.so.1.4.1 differ diff --git a/lib/lz4.c b/lib/lz4.c new file mode 100644 index 0000000..fb84955 --- /dev/null +++ b/lib/lz4.c @@ -0,0 +1,1365 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +/************************************** + Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#define HEAPMODE 0 + +/* + * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : + * By default, the source code expects the compiler to correctly optimize + * 4-bytes and 8-bytes read on architectures able to handle it efficiently. + * This is not always the case. In some circumstances (ARM notably), + * the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses. + * + * You can force the compiler to use unaligned memory access by uncommenting the line below. + * One of the below scenarios will happen : + * 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case). + * You will witness large performance improvements (+50% and up). + * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) + * The goal is to automatically detect such situations by adding your target CPU within an exception list. + * 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler + * No change will be experienced. + * 3 - Your target CPU inefficiently handle unaligned access. + * You will experience a performance loss. Comment back the line. + * 4 - Your target CPU does not handle unaligned access. + * Program will crash. + * If uncommenting results in better performance (case 1) + * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) + * An automatic detection macro will be added to match your case within future versions of the library. + */ +/* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ + + +/************************************** + CPU Feature Detection +**************************************/ +/* + * Automated efficient unaligned memory access detection + * Based on known hardware architectures + * This list will be updated thanks to feedbacks + */ +#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ + || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(__i386__) || defined(__x86_64__) \ + || defined(_M_IX86) || defined(_M_X64) \ + || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ + || (defined(_M_ARM) && (_M_ARM >= 7)) +# define LZ4_UNALIGNED_ACCESS 1 +#else +# define LZ4_UNALIGNED_ACCESS 0 +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +/************************************** + Compiler Options +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +/* "restrict" is a known keyword */ +#else +# define restrict /* Disable restrict */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +/************************************** + Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/************************************** + Includes +**************************************/ +#include "lz4.h" + + +/************************************** + Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** + Reading and writing into memory +**************************************/ +#define STEPSIZE sizeof(size_t) + +static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + return *(U16*)memPtr; + else + { + const BYTE* p = memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + { + *(U16*)memPtr = value; + return; + } + else + { + BYTE* p = memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + + +static U16 LZ4_read16(const void* memPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + return *(U16*)memPtr; + else + { + U16 val16; + memcpy(&val16, memPtr, 2); + return val16; + } +} + +static U32 LZ4_read32(const void* memPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + return *(U32*)memPtr; + else + { + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; + } +} + +static U64 LZ4_read64(const void* memPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + return *(U64*)memPtr; + else + { + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; + } +} + +static size_t LZ4_read_ARCH(const void* p) +{ + if (LZ4_64bits()) + return (size_t)LZ4_read64(p); + else + return (size_t)LZ4_read32(p); +} + + +static void LZ4_copy4(void* dstPtr, const void* srcPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + { + *(U32*)dstPtr = *(U32*)srcPtr; + return; + } + memcpy(dstPtr, srcPtr, 4); +} + +static void LZ4_copy8(void* dstPtr, const void* srcPtr) +{ + if (LZ4_UNALIGNED_ACCESS) + { + if (LZ4_64bits()) + *(U64*)dstPtr = *(U64*)srcPtr; + else + ((U32*)dstPtr)[0] = ((U32*)srcPtr)[0], + ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; + return; + } + memcpy(dstPtr, srcPtr, 8); +} + +/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = dstPtr; + const BYTE* s = srcPtr; + BYTE* e = dstEnd; + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } + else /* Big Endian CPU */ + { + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + +static int LZ4_compress_generic( + void* ctx, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive outputLimited, + tableType_t tableType, + dict_directive dict, + dictIssue_directive dictIssue) +{ + LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; + + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - dictPtr->dictSize; + const BYTE* const dictionary = dictPtr->dictionary; + const BYTE* const dictEnd = dictionary + dictPtr->dictSize; + const size_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + const int skipStrength = SKIPSTRENGTH; + U32 forwardH; + size_t refDelta=0; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source - dictPtr->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize> skipStrength; + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + ref = LZ4_getPositionOnHash(h, ctx, tableType, base); + if (dict==usingExtDict) + { + if (ref<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip)) + || (LZ4_read32(ref+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; /* Check output limit */ + if (litLength>=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; + matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit); + ip += MINMATCH + matchLength; + if (ip==limit) + { + unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchLength += more; + ip += more; + } + } + else + { + matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit); + ip += MINMATCH + matchLength; + } + + if (matchLength>=ML_MASK) + { + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) + return 0; /* Check output limit */ + *token += ML_MASK; + matchLength -= ML_MASK; + for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } + if (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + ref = LZ4_getPosition(ip, ctx, tableType, base); + if (dict==usingExtDict) + { + if (ref<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + LZ4_putPosition(ip, ctx, tableType, base); + if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1) + && (ref+MAX_DISTANCE>=ip) + && (LZ4_read32(ref+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} + +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return (0); +} + + +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ + + if (dictSize < MINMATCH) + { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-MINMATCH) + { + LZ4_putPosition(p, dict, byU32, base); + p+=3; + } + + return dict->dictSize; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ + { + /* rescale hash table */ + U32 delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, + int maxOutputSize, limitedOutput_directive limit) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) + { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); +} + +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); +} + + +/* Hidden debug function, to force separate dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + + result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + + +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; + + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/**************************** + Decompression functions +****************************/ +/* + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is essential this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ4_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* restrict ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; + const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + + /* Main Loop */ + while (1) + { + unsigned token; + size_t length; + const BYTE* match; + + /* get literal length */ + token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) + { + unsigned s; + do + { + s = *ip++; + length += s; + } + while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-COPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + match = cpy - LZ4_readLE16(ip); ip+=2; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) + { + unsigned s; + do + { + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + s = *ip++; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) + { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) + { + /* match can be copied as a single segment from external dictionary */ + match = dictEnd - (lowPrefix-match); + memcpy(op, match, length); + op += length; + } + else + { + /* match encompass external dictionary and current segment */ + size_t copySize = (size_t)(lowPrefix-match); + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + copySize = length - copySize; + if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ + { + BYTE* const endOfMatch = op + copySize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } + else + { + memcpy(op, lowPrefix, copySize); + op += copySize; + } + } + continue; + } + + /* copy repeated sequence */ + cpy = op + length; + if (unlikely((op-match)<8)) + { + const size_t dec64 = dec64table[op-match]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[op-match]; + LZ4_copy4(op+4, match); + op += 8; match -= dec64; + } else { LZ4_copy8(op, match); op+=8; match+=8; } + + if (unlikely(cpy>oend-12)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ + if (op < oend-8) + { + LZ4_wildCopy(op, match, oend-8); + match += (oend-8) - op; + op = oend-8; + } + while (opprefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); +} + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); +} + + +/*************************************************** + Obsolete Functions +***************************************************/ +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } + +static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base) +{ + MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); + lz4ds->bufferStart = base; +} + +int LZ4_resetStreamState(void* state, const char* inputBuffer) +{ + if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (const char* inputBuffer) +{ + void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); + return lz4ds; +} + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data; + + LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB); + + return (char*)(lz4ds->bufferStart + 64 KB); +} + +/* Obsolete compresson functions using User-allocated state */ + +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) +{ + if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ + MEM_INIT(state, 0, LZ4_STREAMSIZE); + + if (inputSize < (int)LZ4_64KLIMIT) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); +} + +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ + MEM_INIT(state, 0, LZ4_STREAMSIZE); + + if (inputSize < (int)LZ4_64KLIMIT) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); +} + +/* Obsolete streaming decompression functions */ + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +#endif /* LZ4_COMMONDEFS_ONLY */ + diff --git a/lib/lz4.h b/lib/lz4.h new file mode 100644 index 0000000..22bbcb5 --- /dev/null +++ b/lib/lz4.h @@ -0,0 +1,310 @@ +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lz4.h provides raw compression format functions, for optimal performance and integration into programs. + * If you need to generate data using an inter-operable format (respecting the framing specification), + * please use lz4frame.h instead. +*/ + +/************************************** + Version +**************************************/ +#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ +#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) +int LZ4_versionNumber (void); + +/************************************** + Tuning parameter +**************************************/ +/* + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#define LZ4_MEMORY_USAGE 14 + + +/************************************** + Simple Functions +**************************************/ + +int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + +/* +LZ4_compress() : + Compresses 'sourceSize' bytes from 'source' into 'dest'. + Destination buffer must be already allocated, + and must be sized to handle worst cases situations (input data not compressible) + Worst case size evaluation is provided by function LZ4_compressBound() + inputSize : Max supported value is LZ4_MAX_INPUT_SIZE + return : the number of bytes written in buffer dest + or 0 if the compression fails + +LZ4_decompress_safe() : + compressedSize : is obviously the source size + maxDecompressedSize : is the size of the destination buffer, which must be already allocated. + return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) + If the destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, + and never writes outside of output buffer, nor reads outside of input buffer. + It is also protected against malicious data packets. +*/ + + +/************************************** + Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/* +LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (output buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + + isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ +int LZ4_compressBound(int isize); + + +/* +LZ4_compress_limitedOutput() : + Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. + If it cannot achieve it, compression will stop, and result of the function will be zero. + This saves time and memory on detecting non-compressible (or barely compressible) data. + This function never writes outside of provided output buffer. + + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxOutputSize : is the size of the destination buffer (which must be already allocated) + return : the number of bytes written in buffer 'dest' + or 0 if compression fails +*/ +int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); + + +/* +LZ4_compress_withState() : + Same compression functions, but using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and then, provide it as 'void* state' to compression functions. +*/ +int LZ4_sizeofState(void); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); + + +/* +LZ4_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +int LZ4_decompress_fast (const char* source, char* dest, int originalSize); + + +/* +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*********************************************** + Streaming Compression Functions +***********************************************/ + +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +/* + * LZ4_stream_t + * information structure to track an LZ4 stream. + * important : init this structure content before first use ! + */ +typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; + +/* + * LZ4_resetStream + * Use this function to init an allocated LZ4_stream_t structure + */ +void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); + +/* + * If you prefer dynamic allocation methods, + * LZ4_createStream will allocate and initialize an LZ4_stream_t structure + * LZ4_freeStream releases its memory. + */ +LZ4_stream_t* LZ4_createStream(void); +int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); + +/* + * LZ4_loadDict + * Use this function to load a static dictionary into LZ4_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); + +/* + * LZ4_compress_continue + * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio + * Previous data blocks are assumed to still be present at their previous location. + */ +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); + +/* + * LZ4_compress_limitedOutput_continue + * Same as before, but also specify a maximum target compressed size (maxOutputSize) + * If objective cannot be met, compression exits, and returns a zero. + */ +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* + * LZ4_saveDict + * If previously compressed data block is not guaranteed to remain available at its memory location + * save it into a safer place (char* safeBuffer) + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() + * Return : dictionary size in bytes, or 0 if error + * Note : any dictSize > 64 KB will be interpreted as 64KB. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); + + +/************************************************ + Streaming Decompression Functions +************************************************/ + +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +/* + * LZ4_streamDecode_t + * information structure to track an LZ4 stream. + * init this structure content using LZ4_setStreamDecode or memset() before first use ! + * If you prefer dynamic allocation methods : + * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure + * LZ4_freeStreamDecode releases its memory. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void); +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/* + * LZ4_setStreamDecode + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * Return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + If this condition is not possible, save the relevant part of decoded data into a safe buffer, + and indicate where is its new address using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as + a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() + They are stand-alone and don't use nor update an LZ4_streamDecode_t structure. +*/ +int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + + +/************************************** + Obsolete Functions +**************************************/ +/* +Obsolete decompression functions +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is the same as LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe +These function prototypes are now disabled; uncomment them if you really need them. +It is highly recommended to stop using these functions and migrate to newer ones */ +/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ +/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ + + +/* Obsolete streaming functions; use new streaming interface whenever possible */ +void* LZ4_create (const char* inputBuffer); +int LZ4_sizeofStreamState(void); +int LZ4_resetStreamState(void* state, const char* inputBuffer); +char* LZ4_slideInputBuffer (void* state); + +/* Obsolete streaming decoding functions */ +int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize); +int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); + + +#if defined (__cplusplus) +} +#endif diff --git a/lib/lz4.o b/lib/lz4.o new file mode 100644 index 0000000..2ed244d Binary files /dev/null and b/lib/lz4.o differ diff --git a/lib/lz4frame.c b/lib/lz4frame.c new file mode 100644 index 0000000..aa46152 --- /dev/null +++ b/lib/lz4frame.c @@ -0,0 +1,1330 @@ +/* +LZ4 auto-framing library +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API to create LZ4-compressed Frames +* fully conformant to specification v1.4.1. +* All related operations, including memory management, are handled by the library. +* */ + + +/************************************** +Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + +/************************************** +Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(s) calloc(1,s) +#define FREEMEM free +#include /* memset, memcpy, memmove */ +#define MEM_INIT memset + + +/************************************** +Includes +**************************************/ +#include "lz4frame.h" +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" + + +/************************************** +Basic Types +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + + +/************************************** +Constants +**************************************/ +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ4F_MAXHEADERFRAME_SIZE 7 +#define LZ4F_BLOCKSIZEID_DEFAULT max64KB + +static const U32 minHClevel = 3; + +/************************************** +Structures and local types +**************************************/ +typedef struct +{ + LZ4F_preferences_t prefs; + U32 version; + U32 cStage; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; + BYTE* tmpIn; + size_t tmpInSize; + XXH32_state_t xxh; + void* lz4CtxPtr; + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ +} LZ4F_cctx_internal_t; + +typedef struct +{ + LZ4F_frameInfo_t frameInfo; + unsigned version; + unsigned dStage; + size_t maxBlockSize; + size_t maxBufferSize; + const BYTE* srcExpect; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + BYTE header[8]; +} LZ4F_dctx_internal_t; + + +/************************************** +Macros +**************************************/ + + +/************************************** +Error management +**************************************/ +#define LZ4F_GENERATE_STRING(STRING) #STRING, +static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; + + +int LZ4F_isError(LZ4F_errorCode_t code) +{ + return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); +} + +const char* LZ4F_getErrorName(LZ4F_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; + return codeError; +} + + +/************************************** +Private functions +**************************************/ +static size_t LZ4F_getBlockSize(unsigned blockSizeID) +{ + static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; + + if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + blockSizeID -= 4; + if (blockSizeID > 3) return (size_t)-ERROR_maxBlockSize_invalid; + return blockSizes[blockSizeID]; +} + + +/* unoptimized version; solves endianess & alignment issues */ +static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) +{ + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U32 LZ4F_readLE32 (const BYTE* srcPtr) +{ + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += (srcPtr[3]<<24); + return value32; +} + + +static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) +{ + U32 xxh = XXH32(header, (U32)length, 0); + return (BYTE)(xxh >> 8); +} + + +/************************************** +Simple compression functions +**************************************/ +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs = { 0 }; + size_t headerSize; + size_t streamSize; + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + { + blockSizeID_t proposedBSID = max64KB; + size_t maxBlockSize = 64 KB; + while (prefs.frameInfo.blockSizeID > proposedBSID) + { + if (srcSize <= maxBlockSize) + { + prefs.frameInfo.blockSizeID = proposedBSID; + break; + } + proposedBSID++; + maxBlockSize <<= 2; + } + } + prefs.autoFlush = 1; + + headerSize = 7; /* basic header size (no option) including magic number */ + streamSize = LZ4F_compressBound(srcSize, &prefs); + + return headerSize + streamSize; +} + + +/* LZ4F_compressFrame() +* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() +* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) +* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +* The result of the function is the number of bytes written into dstBuffer. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ + LZ4F_preferences_t prefs = { 0 }; + LZ4F_compressOptions_t options = { 0 }; + LZ4F_errorCode_t errorCode; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstMaxSize; + + + cctxI.version = LZ4F_VERSION; + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + { + blockSizeID_t proposedBSID = max64KB; + size_t maxBlockSize = 64 KB; + while (prefs.frameInfo.blockSizeID > proposedBSID) + { + if (srcSize <= maxBlockSize) + { + prefs.frameInfo.blockSizeID = proposedBSID; + break; + } + proposedBSID++; + maxBlockSize <<= 2; + } + } + prefs.autoFlush = 1; + if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */ + + options.stableSrc = 1; + + if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) + return (size_t)-ERROR_dstMaxSize_tooSmall; + + errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; /* header size */ + + dstMaxSize -= errorCode; + errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + FREEMEM(cctxI.lz4CtxPtr); + + return (dstPtr - dstStart); +} + + +/*********************************** +* Advanced compression functions +* *********************************/ + +/* LZ4F_createCompressionContext() : +* The first thing to do is to create a compressionContext object, which will be used in all compression operations. +* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. +* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. +* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LZ4F_freeCompressionContext(); +*/ +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) +{ + LZ4F_cctx_internal_t* cctxPtr; + + cctxPtr = (LZ4F_cctx_internal_t*)ALLOCATOR(sizeof(LZ4F_cctx_internal_t)); + if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-ERROR_allocation_failed); + + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Next stage : write header */ + + *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; + + return OK_NoError; +} + + +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) +{ + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; + + FREEMEM(cctxPtr->lz4CtxPtr); + FREEMEM(cctxPtr->tmpBuff); + FREEMEM(LZ4F_compressionContext); + + return OK_NoError; +} + + +/* LZ4F_compressBegin() : +* will write the frame header into dstBuffer. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. +* The result of the function is the number of bytes written into dstBuffer for the header +* or an error code (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* headerStart; + size_t requiredBuffSize; + + if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctxPtr->prefs = *preferencesPtr; + + /* ctx Management */ + { + U32 targetCtxLevel = cctxPtr->prefs.compressionLevellz4CtxLevel < targetCtxLevel) + { + FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr->prefs.compressionLevellz4CtxPtr = (void*)LZ4_createStream(); + else + cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); + cctxPtr->lz4CtxLevel = targetCtxLevel; + } + } + + /* Buffer Management */ + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 128 KB); + if (preferencesPtr->autoFlush) + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 64 KB; /* just needs dict */ + + if (cctxPtr->maxBufferSize < requiredBuffSize) + { + cctxPtr->maxBufferSize = requiredBuffSize; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return (size_t)-ERROR_allocation_failed; + } + cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpInSize = 0; + XXH32_reset(&(cctxPtr->xxh), 0); + if (cctxPtr->prefs.compressionLevellz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + + /* Magic Number */ + LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); + dstPtr += 4; + headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ + /* BD Byte */ + *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* CRC Byte */ + *dstPtr++ = LZ4F_headerChecksum(headerStart, 2); + + cctxPtr->cStage = 1; /* header written, wait for data block */ + + return (dstPtr - dstStart); +} + + +/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. +* The LZ4F_frameInfo_t structure is optional : +* you can provide NULL as argument, all preferences will then be set to default. +* */ +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + const LZ4F_preferences_t prefsNull = { 0 }; + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; + + return result; +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); + +static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) +{ + /* compress one block */ + BYTE* cSizePtr = (BYTE*)dst; + U32 cSize; + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = (U32)srcSize; + LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+4, src, srcSize); + } + return cSize + 4; +} + + +static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); +} + +static compressFunc_t LZ4F_selectCompression(blockMode_t blockMode, U32 level) +{ + if (level < minHClevel) + { + if (blockMode == blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; + return LZ4F_localLZ4_compress_limitedOutput_continue; + } + if (blockMode == blockIndependent) return LZ4_compressHC2_limitedOutput_withStateHC; + return LZ4F_localLZ4_compressHC_limitedOutput_continue; +} + +static int LZ4F_localSaveDict(LZ4F_cctx_internal_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < minHClevel) + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; + +/* LZ4F_compressUpdate() +* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + size_t blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LZ4F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t compress; + + + if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; + if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ + { + size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; + if (sizeToCopy > srcSize) + { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } + else + { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + + if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) + { + /* compress full block */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) + { + /* compress remaining input < blockSize */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr = srcEnd; + } + + /* preserve dictionary if necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer)) + { + if (compressOptionsPtr->stableSrc) + { + cctxPtr->tmpIn = cctxPtr->tmpBuff; + } + else + { + int realDictSize = LZ4F_localSaveDict(cctxPtr); + if (realDictSize==0) return (size_t)-ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ + && !(cctxPtr->prefs.autoFlush)) + { + LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) + { + /* fill tmp buffer */ + size_t sizeToCopy = srcEnd - srcPtr; + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize); + + return dstPtr - dstStart; +} + + +/* LZ4F_flush() +* Should you need to create compressed data immediately, without waiting for a block to be filled, +* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. +* The result of the function is the number of bytes written into dstBuffer +* (it can be zero, this means there was no data left within compressionContext) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +*/ +size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + (void)compressOptionsPtr; /* not yet useful */ + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* compress tmp buffer */ + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */ + { + LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; + } + + return dstPtr - dstStart; +} + + +/* LZ4F_compressEnd() +* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). +* It will flush whatever data remained within compressionContext (like LZ4_flush()) +* but also properly finalize the frame, with an endMark and a checksum. +* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. +*/ +size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + size_t errorCode; + + errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LZ4F_writeLE32(dstPtr, 0); + dstPtr+=4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + { + U32 xxh = XXH32_digest(&(cctxPtr->xxh)); + LZ4F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + return dstPtr - dstStart; +} + + +/*********************************** +* Decompression functions +* *********************************/ + +/* Resource management */ + +/* LZ4F_createDecompressionContext() : +* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. +* This is achieved using LZ4F_createDecompressionContext(). +* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. +* If the result LZ4F_errorCode_t is not zero, there was an error during context creation. +* Object can release its memory using LZ4F_freeDecompressionContext(); +*/ +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) +{ + LZ4F_dctx_internal_t* dctxPtr; + + dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); + if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; + + dctxPtr->version = versionNumber; + *LZ4F_decompressionContextPtr = (LZ4F_compressionContext_t)dctxPtr; + return OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext) +{ + LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext; + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + FREEMEM(dctxPtr); + return OK_NoError; +} + + +/* Decompression */ + +static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize) +{ + BYTE FLG, BD, HC; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; + size_t bufferNeeded; + + /* need to decode header to get frameInfo */ + if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ + + /* control magic number */ + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC; + srcPtr += 4; + + /* Flags */ + FLG = srcPtr[0]; + version = (FLG>>6)&_2BITS; + blockMode = (FLG>>5) & _1BIT; + blockChecksumFlag = (FLG>>4) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + dictFlag = (FLG>>0) & _1BIT; + BD = srcPtr[1]; + blockSizeID = (BD>>4) & _3BITS; + + /* check */ + HC = LZ4F_headerChecksum(srcPtr, 2); + if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ + + /* validate */ + if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ + if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ + if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + + /* save */ + dctxPtr->frameInfo.blockMode = blockMode; + dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = blockSizeID; + dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + + /* init */ + if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); + + /* alloc */ + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==blockLinked) * 128 KB); + if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ + { + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + dctxPtr->maxBufferSize = bufferNeeded; + dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); + if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; + dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 0; + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = 0; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + dctxPtr->tmpOutStart = 0; + dctxPtr->tmpOutSize = 0; + + return 7; +} + + +typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader, + dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, + dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix +} dStage_t; + + +/* LZ4F_getFrameInfo() +* This function decodes frame header information, such as blockSize. +* It is optional : you could start by calling directly LZ4F_decompress() instead. +* The objective is to extract header information without starting decompression, typically for allocation purposes. +* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. +* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, +* or an error code which can be tested using LZ4F_isError(). +*/ +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) +{ + LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; + + if (dctxPtr->dStage == dstage_getHeader) + { + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr); + if (LZ4F_isError(errorCode)) return errorCode; + *srcSizePtr = errorCode; + *frameInfoPtr = dctxPtr->frameInfo; + dctxPtr->srcExpect = NULL; + dctxPtr->dStage = dstage_getCBlockSize; + return 4; + } + + /* frameInfo already decoded */ + *srcSizePtr = 0; + *frameInfoPtr = dctxPtr->frameInfo; + return 0; +} + + +static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +{ + (void)dictStart; + (void)dictSize; + return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); +} + + + +static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +{ + if (dctxPtr->dictSize==0) + dctxPtr->dict = (BYTE*)dstPtr; /* priority to dictionary continuity */ + + if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ + { + dctxPtr->dictSize += dstSize; + return; + } + + if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ + { + dctxPtr->dict = (BYTE*)dstPtr0; + dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + return; + } + + if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) + { + /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ + dctxPtr->dictSize += dstSize; + return; + } + + if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + { +#if 0 + size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize; + return; + +#else + + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + return; +#endif + } + + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ + { + if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + dctxPtr->dictSize = preserveSize; + } + memcpy(dctxPtr->dict + dctxPtr->dictSize, dstPtr, dstSize); + dctxPtr->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dstSize; + } +} + + + +/* LZ4F_decompress() +* Call this function repetitively to regenerate data compressed within srcBuffer. +* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* +* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). +* +* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. +* You will have to call it again, continuing from where it stopped. +* +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. +* Basically, it's the size of the current (or remaining) compressed block + header of next block. +* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. +* Note that this is just a hint, you can always provide any srcSize you want. +* When a frame is fully decoded, the function result will be 0. +* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). +*/ +size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; + static const LZ4F_decompressOptions_t optionsNull = { 0 }; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn=NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + + /* expect to continue decoding src buffer where it left previously */ + if (dctxPtr->srcExpect != NULL) + { + if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_GENERIC; + } + + /* programmed as a state machine */ + + while (doAnotherStage) + { + + switch(dctxPtr->dStage) + { + + case dstage_getHeader: + { + if (srcEnd-srcPtr >= 7) + { + selectedIn = srcPtr; + srcPtr += 7; + dctxPtr->dStage = dstage_decodeHeader; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeHeader; + break; + } + + case dstage_storeHeader: + { + size_t sizeToCopy = 7 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < 7) + { + nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; + doAnotherStage = 0; /* no enough src, wait to get some more */ + break; + } + selectedIn = dctxPtr->header; + dctxPtr->dStage = dstage_decodeHeader; + break; + } + + case dstage_decodeHeader: + { + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7); + if (LZ4F_isError(errorCode)) return errorCode; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + + case dstage_getCBlockSize: + { + if ((srcEnd - srcPtr) >= 4) + { + selectedIn = srcPtr; + srcPtr += 4; + dctxPtr->dStage = dstage_decodeCBlockSize; + break; + } + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + break; + } + + case dstage_storeCBlockSize: + { + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlockSize; + break; + } + + case dstage_decodeCBlockSize: + { + size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; + if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ + { + dctxPtr->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */ + dctxPtr->tmpInTarget = nextCBlockSize; + if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) + { + dctxPtr->dStage = dstage_copyDirect; + break; + } + dctxPtr->dStage = dstage_getCBlock; + if (dstPtr==dstEnd) + { + nextSrcSizeHint = nextCBlockSize + 4; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + { + size_t sizeToCopy = dctxPtr->tmpInTarget; + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, (U32)sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ + nextSrcSizeHint = dctxPtr->tmpInTarget + 4; + doAnotherStage = 0; + break; + } + + case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ + { + if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlock; + break; + } + selectedIn = srcPtr; + srcPtr += dctxPtr->tmpInTarget; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_storeCBlock: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_decodeCBlock: + { + if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + else + dctxPtr->dStage = dstage_decodeCBlock_intoDst; + break; + } + + case dstage_decodeCBlock_intoDst: + { + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; + + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + + dstPtr += decodedSize; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + + case dstage_decodeCBlock_intoTmp: + { + /* not enough place into dst : decode into tmpOut */ + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; + + /* ensure enough place for tmpOut */ + if (dctxPtr->frameInfo.blockMode == blockLinked) + { + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) + { + if (dctxPtr->dictSize > 128 KB) + { + memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); + dctxPtr->dictSize = 64 KB; + } + dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize; + } + else /* dict not within tmp */ + { + size_t reservedDictSpace = dctxPtr->dictSize; + if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + } + } + + /* Decode */ + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + dctxPtr->tmpOutSize = decodedSize; + dctxPtr->tmpOutStart = 0; + dctxPtr->dStage = dstage_flushOut; + break; + } + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + { + size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; + if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; + memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + + dctxPtr->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + + /* end of flush ? */ + if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + nextSrcSizeHint = 4; + doAnotherStage = 0; /* still some data to flush */ + break; + } + + case dstage_getSuffix: + { + size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; + if (suffixSize == 0) /* frame completed */ + { + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) >= 4) /* CRC present */ + { + selectedIn = srcPtr; + srcPtr += 4; + dctxPtr->dStage = dstage_checkSuffix; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + break; + } + + case dstage_storeSuffix: + { + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_checkSuffix; + break; + } + + case dstage_checkSuffix: + { + U32 readCRC = LZ4F_readLE32(selectedIn); + U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); + if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + } + } + + /* preserve dictionary within tmp if necessary */ + if ( (dctxPtr->frameInfo.blockMode==blockLinked) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + &&(!decompressOptionsPtr->stableDst) + &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) + ) + { + if (dctxPtr->dStage == dstage_flushOut) + { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + } + else + { + size_t newDictSize = dctxPtr->dictSize; + BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + if ((newDictSize) > 64 KB) newDictSize = 64 KB; + + memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = newDictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + } + } + + if (srcPtrsrcExpect = srcPtr; + else + dctxPtr->srcExpect = NULL; + *srcSizePtr = (srcPtr - srcStart); + *dstSizePtr = (dstPtr - dstStart); + return nextSrcSizeHint; +} diff --git a/lib/lz4frame.h b/lib/lz4frame.h new file mode 100644 index 0000000..a4f5e36 --- /dev/null +++ b/lib/lz4frame.h @@ -0,0 +1,261 @@ +/* + LZ4 auto-framing library + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API to create LZ4-compressed frames + * fully conformant to specification v1.4.1. + * All related operations, including memory management, are handled by the library. + * You don't need lz4.h when using lz4frame.h. + * */ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/**************************************** + Note : experimental API. + Not yet integrated within lz4 library. +****************************************/ + +/************************************** + Includes +**************************************/ +#include /* size_t */ + + +/************************************** + Error management +**************************************/ +typedef size_t LZ4F_errorCode_t; +#define LZ4F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_checksum_invalid) \ + ITEM(ERROR_maxCode) + +#define LZ4F_GENERATE_ENUM(ENUM) ENUM, +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ + +int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ +const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */ + + +/************************************** + Framing compression functions +**************************************/ + +typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} blockSizeID_t; +typedef enum { blockLinked=0, blockIndependent} blockMode_t; +typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; + +typedef struct { + blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + unsigned reserved[5]; +} LZ4F_frameInfo_t; + +typedef struct { + LZ4F_frameInfo_t frameInfo; + unsigned compressionLevel; /* Not yet supported : only fast compression for the time being */ + unsigned autoFlush; /* 1 == always flush; reduce need for tmp buffer */ + unsigned reserved[4]; +} LZ4F_preferences_t; + + + +/*********************************** + * Simple compression function + * *********************************/ +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); + +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); +/* LZ4F_compressFrame() + * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() + * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + + + +/********************************** + * Advanced compression functions + * *********************************/ + +typedef void* LZ4F_compressionContext_t; + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned reserved[3]; +} LZ4F_compressOptions_t; + +/* Resource Management */ + +#define LZ4F_VERSION 100 +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext); +/* LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); + */ + + +/* Compression */ + +size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr); +/* LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes. + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The result of the function is the number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); +/* LZ4F_compressBound() : + * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. + * preferencesPtr is optional : you can provide NULL as argument, all preferences will then be set to default. + */ + +size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); +/* LZ4F_compressUpdate() + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) + * You can get the minimum value of dstMaxSize by using LZ4F_compressBound() + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); +/* LZ4F_flush() + * Should you need to create compressed data immediately, without waiting for a block to be filled, + * you can call LZ4_flush(), which will immediately compress any remaining data buffered within compressionContext. + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within compressionContext) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); +/* LZ4F_compressEnd() + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * compressionContext can then be used again, starting with LZ4F_compressBegin(). + */ + + +/*********************************** + * Decompression functions + * *********************************/ + +typedef void* LZ4F_decompressionContext_t; + +typedef struct { + unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ + unsigned reserved[3]; +} LZ4F_decompressOptions_t; + +/* Resource management */ + +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext); +/* LZ4F_createDecompressionContext() : + * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. + * This is achieved using LZ4F_createDecompressionContext(). + * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeDecompressionContext(); + */ + +/* Decompression */ + +size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); +/* LZ4F_getFrameInfo() + * This function decodes frame header information, such as blockSize. + * It is optional : you could start by calling directly LZ4F_decompress() instead. + * The objective is to extract header information without starting decompression, typically for allocation purposes. + * LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. + * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, + * or an error code which can be tested using LZ4F_isError(). + */ + +size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr); +/* LZ4F_decompress() + * Call this function repetitively to regenerate data compressed within srcBuffer. + * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes effectively used from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. + * This typically happens when dstBuffer is not large enough to contain all decoded data. + * LZ4F_decompress() will have to be called again, starting from where it stopped (srcBuffer + *srcSizePtr) + * The function will check this condition, and refuse to continue if it is not respected. + * dstBuffer is supposed to be flushed between calls to the function, since its content will be rewritten. + * Different dst arguments can be used between each calls. + * + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Basically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, you can always provide any srcSize you want. + * When a frame is fully decoded, the function result will be 0. + * If decompression failed, function result is an error code which can be tested using LZ4F_isError(). + */ + + + +#if defined (__cplusplus) +} +#endif diff --git a/lib/lz4hc.c b/lib/lz4hc.c new file mode 100644 index 0000000..89f0db0 --- /dev/null +++ b/lib/lz4hc.c @@ -0,0 +1,768 @@ +/* +LZ4 HC - High Compression Mode of LZ4 +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html +- LZ4 source repository : http://code.google.com/p/lz4/ +*/ + + + +/************************************** + Tuning Parameter +**************************************/ +#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 + + +/************************************** + Includes +**************************************/ +#include "lz4.h" +#include "lz4hc.h" + + +/************************************** + Local Compiler Options +**************************************/ +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +#if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# pragma warning(disable : 4201) /* disable: C4201: unnamed struct/union*/ +#endif + + +/************************************** + Common LZ4 definition +**************************************/ +#define LZ4_COMMONDEFS_ONLY +#include "lz4.c" + + +/************************************** + Local Constants +**************************************/ +#define DICTIONARY_LOGSIZE 16 +#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) +#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] +#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) + +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } + + + +/************************************** + HC Compression +**************************************/ +static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) +{ + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); + hc4->nextToUpdate = 64 KB; + hc4->base = base - 64 KB; + hc4->inputBuffer = base; + hc4->end = base; + hc4->dictBase = base - 64 KB; + hc4->dictLimit = 64 KB; + hc4->lowLimit = 64 KB; +} + + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +{ + U16* chainTable = hc4->chainTable; + U32* HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const U32 target = (U32)(ip - base); + U32 idx = hc4->nextToUpdate; + + while(idx < target) + { + U32 h = LZ4HC_hashPtr(base+idx); + size_t delta = idx - HashTable[h]; + if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; + chainTable[idx & 0xFFFF] = (U16)delta; + HashTable[h] = idx; + idx++; + } + + hc4->nextToUpdate = target; +} + + +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + // Note : need to handle risk of index overflow + // Use only one memory segment for dict, so any previous External Dict is lost at this stage + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block +} + + +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts) +{ + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const BYTE* const dictBase = hc4->dictBase; + const U32 dictLimit = hc4->dictLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + U32 matchIndex; + const BYTE* match; + int nbAttempts=maxNbAttempts; + size_t ml=0; + + /* HC4 match finder */ + LZ4HC_Insert(hc4, ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + + while ((matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) + && (LZ4_read32(match) == LZ4_read32(ip))) + { + size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + size_t mlt; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos + } + } + matchIndex -= chainTable[matchIndex & 0xFFFF]; + } + + return (int)ml; +} + + +FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_Data_Structure* hc4, + const BYTE* ip, + const BYTE* iLowLimit, + const BYTE* iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts) +{ + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const U32 dictLimit = hc4->dictLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const BYTE* const dictBase = hc4->dictBase; + const BYTE* match; + U32 matchIndex; + int nbAttempts = maxNbAttempts; + int delta = (int)(ip-iLowLimit); + + + /* First Match */ + LZ4HC_Insert(hc4, ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + + while ((matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(iLowLimit + longest) == *(match - delta + longest)) + if (LZ4_read32(match) == LZ4_read32(ip)) + { + const BYTE* startt = ip; + const BYTE* tmpMatch = match; + const BYTE* const matchEnd = ip + MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + + while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} + + if ((matchEnd-startt) > longest) + { + longest = (int)(matchEnd-startt); + *matchpos = tmpMatch; + *startpos = startt; + } + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + } + } + matchIndex -= chainTable[matchIndex & 0xFFFF]; + } + + return longest; +} + + +typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; + +//static unsigned debug = 0; + +FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limitedOutputBuffer, + BYTE* oend) +{ + int length; + BYTE* token; + + //if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); // debug + + /* Encode Literal length */ + length = (int)(*ip - *anchor); + token = (*op)++; + if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } + else *token += (BYTE)(length); + + /* Prepare next loop */ + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +#define MAX_COMPRESSION_LEVEL 16 +static int LZ4HC_compress_generic ( + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) +{ + LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + unsigned maxNbAttempts; + int ml, ml2, ml3, ml0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* start3=NULL; + const BYTE* ref3=NULL; + const BYTE* start0; + const BYTE* ref0; + + + /* init */ + if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; + if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; + maxNbAttempts = 1 << compressionLevel; + ctx->end += inputSize; + + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); + if (!ml) { ip++; continue; } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); + else ml2 = ml; + + if (ml2 == ml) /* No better match */ + { + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + continue; + } + + if (start0 < ip) + { + if (start2 < ip + ml0) /* empirical */ + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) /* First Match too small : removed */ + { + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + /* + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ + if ((start2 - ip) < OPTIMAL_ML) + { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ + + if (start2 + ml2 < mflimit) + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); + else ml3 = ml2; + + if (ml3 == ml2) /* No better match : 2 sequences to encode */ + { + /* ip & ref are known; Now for ml */ + if (start2 < ip+ml) ml = (int)(start2 - ip); + /* Now, encode 2 sequences */ + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start2; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; + continue; + } + + if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ + { + if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + { + if (start2 < ip+ml) + { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) + { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ + if (start2 < ip+ml) + { + if ((start2 - ip) < (int)ML_MASK) + { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + else + { + ml = (int)(start2 - ip); + } + } + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<base = NULL; + ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; +} + +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) +{ + LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + if (dictSize > 64 KB) + { + dictionary += dictSize - 64 KB; + dictSize = 64 KB; + } + LZ4HC_init (streamPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ4HC_Insert (streamPtr, (const BYTE*)dictionary +(dictSize-3)); + streamPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; +} + + +/* compression */ + +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize, limitedOutput_directive limit) +{ + /* auto-init if forgotten */ + if (dsPtr->base == NULL) + LZ4HC_init (dsPtr, (const BYTE*) source); + + /* Check overflow */ + if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB) + { + size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit; + if (dictSize > 64 KB) dictSize = 64 KB; + + LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; + const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) + { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); + if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; + } + } + + return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); +} + +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) +{ + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); +} + +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); +} + + +/* dictionary saving */ + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); + //LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); + { + U32 endIndex = (U32)(streamPtr->end - streamPtr->base); + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->base = streamPtr->end - endIndex; + streamPtr->dictLimit = endIndex - dictSize; + streamPtr->lowLimit = endIndex - dictSize; + } + return dictSize; +} + + +/*********************************** + * Deprecated Functions + ***********************************/ +int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } + +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) +{ + if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ + LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); + return 0; +} + +void* LZ4_createHC (const char* inputBuffer) +{ + void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); + LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + FREEMEM(LZ4HC_Data); + return (0); +} + +/* +int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) +{ +return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); +} +int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) +{ +return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); +} +*/ + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; + size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; + + if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ + + distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ + LZ4HC_Insert(hc4, hc4->end - MINMATCH); + memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); + hc4->base -= distance; + if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ + { + int i; + hc4->base += 1 GB; + for (i=0; ihashTable[i] -= 1 GB; + } + hc4->end -= distance; + return (char*)(hc4->end); +} diff --git a/lib/lz4hc.h b/lib/lz4hc.h new file mode 100644 index 0000000..26ba6d3 --- /dev/null +++ b/lib/lz4hc.h @@ -0,0 +1,223 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + + +#if defined (__cplusplus) +extern "C" { +#endif + + +int LZ4_compressHC (const char* source, char* dest, int inputSize); +/* +LZ4_compressHC : + return : the number of bytes in compressed buffer dest + or 0 if compression fails. + note : destination buffer must be already allocated. + To avoid any problem, size it to handle worst cases situations (input data not compressible) + Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") +*/ + +int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +/* +LZ4_compress_limitedOutput() : + Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. + If it cannot achieve it, compression will stop, and result of the function will be zero. + This function never writes outside of provided output buffer. + + inputSize : Max supported value is 1 GB + maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) + return : the number of output bytes written in buffer 'dest' + or 0 if compression fails. +*/ + + +int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +/* + Same functions as above, but with programmable 'compressionLevel'. + Recommended values are between 4 and 9, although any value between 0 and 16 will work. + 'compressionLevel'==0 means use default 'compressionLevel' value. + Values above 16 behave the same as 16. + Equivalent variants exist for all other compression functions below. +*/ + +/* Note : + Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) +*/ + + +/************************************** + Using an external allocation +**************************************/ +int LZ4_sizeofStateHC(void); +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); + +int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); + +/* +These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. +To know how much memory must be allocated for the compression tables, use : +int LZ4_sizeofStateHC(); + +Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). + +The allocated memory can be provided to the compression functions using 'void* state' parameter. +LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. +They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap). +*/ + + + +/************************************** + Experimental Streaming Functions +**************************************/ +#define LZ4_STREAMHCSIZE_U64 32774 +#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ4_STREAMHCSIZE_U64]; } LZ4_streamHC_t; + +/* +This structure allows static allocation of LZ4 HC streaming state. +State must then be initialized using LZ4_resetStreamHC() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +LZ4_streamHC_t* LZ4_createStreamHC(void); +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); + +/* +These functions create and release memory for LZ4 HC streaming state. +Newly created states are already initialized. +Existing state space can be re-used anytime using LZ4_resetStreamHC(). +*/ + +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize); + +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize); + +/* +These functions compress data in successive blocks of any size, using previous blocks as dictionary. +One key assumption is that each previous block will remain read-accessible while compressing next block. + +Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). +A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). + +Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block. +They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. +Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + +If, for any reason, previous data block can't be preserved in memory during next compression block, +you must save it to a safer memory space, +using LZ4_saveDictHC(). +*/ + + + +/************************************** + Deprecated Streaming Functions +**************************************/ +/* Note : these streaming functions still follows the older model */ +void* LZ4_createHC (const char* inputBuffer); +//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); +//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); +char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +int LZ4_freeHC (void* LZ4HC_Data); + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); + +/* +These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. +In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : + +void* LZ4_createHC (const char* inputBuffer); +The result of the function is the (void*) pointer on the LZ4HC Data Structure. +This pointer will be needed in all other functions. +If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. +The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. +To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). +Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), +but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. +If next block does not begin immediately after the previous one, the compression will fail (return 0). + +When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : +char* LZ4_slideInputBufferHC(void* LZ4HC_Data); +must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. +Note that, for this function to work properly, minimum size of an input buffer must be 192KB. +==> The memory position where the next input data block must start is provided as the result of the function. + +Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. + +When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. +*/ + +int LZ4_sizeofStreamStateHC(void); +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); + +/* +These functions achieve the same result as : +void* LZ4_createHC (const char* inputBuffer); + +They are provided here to allow the user program to allocate memory using its own routines. + +To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); +Note also that space must be aligned for pointers (32 or 64 bits). + +Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); +void* state is a pointer to the space allocated. +It must be aligned for pointers (32 or 64 bits), and be large enough. +The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. +The input buffer must be already allocated, and size at least 192KB. +'inputBuffer' will also be the 'const char* source' of the first block. + +The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). +return value of LZ4_resetStreamStateHC() must be 0 is OK. +Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). +*/ + + +#if defined (__cplusplus) +} +#endif diff --git a/lib/lz4hc.o b/lib/lz4hc.o new file mode 100644 index 0000000..bbc0911 Binary files /dev/null and b/lib/lz4hc.o differ diff --git a/lib/xxhash.c b/lib/xxhash.c new file mode 100644 index 0000000..24a64b5 --- /dev/null +++ b/lib/xxhash.c @@ -0,0 +1,928 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +- public discussion board : https://groups.google.com/forum/#!forum/lz4c +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory routines +// for malloc(), free() +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +// for memcpy() +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S +{ + U32 v; +} _PACKED U32_S; +typedef struct _U64_S +{ + U64 v; +} _PACKED U64_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static inline U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static inline U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + else + return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +unsigned int XXH32 (const void* input, size_t len, unsigned seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/**************************************************** + * Advanced Hash Functions +****************************************************/ + +/*** Allocation ***/ +typedef struct +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + U32 mem32[4]; /* defined as U32 for alignment */ + U32 memsize; +} XXH_istate32_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + U64 mem64[4]; /* defined as U64 for alignment */ + U32 memsize; +} XXH_istate64_t; + + +XXH32_state_t* XXH32_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + +XXH64_state_t* XXH64_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + + +/*** Hash feed ***/ + +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +{ + XXH_istate64_t* state = (XXH_istate64_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { + const U32* p32 = state->mem32; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32(p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32(p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32(p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32(p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem32, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const BYTE * p = (const BYTE*)state->mem32; + BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) + { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + { + const U64* p64 = state->mem64; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64(p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64(p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64(p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64(p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem64, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE * p = (const BYTE*)state->mem64; + BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_readLE64(p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + diff --git a/lib/xxhash.h b/lib/xxhash.h new file mode 100644 index 0000000..55b4501 --- /dev/null +++ b/lib/xxhash.h @@ -0,0 +1,156 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +/***************************** + Includes +*****************************/ +#include /* size_t */ + + +/***************************** + Type +*****************************/ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +/***************************** + Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". +*/ + + + +/***************************** + Advanced Hash Functions +*****************************/ +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; + +/* +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ + + +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. + +XXH state space must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +*/ + + +#if defined (__cplusplus) +} +#endif diff --git a/lz4.c b/lz4.c deleted file mode 100644 index fb84955..0000000 --- a/lz4.c +++ /dev/null @@ -1,1365 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - - -/************************************** - Tuning parameters -**************************************/ -/* - * HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ -#define HEAPMODE 0 - -/* - * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : - * By default, the source code expects the compiler to correctly optimize - * 4-bytes and 8-bytes read on architectures able to handle it efficiently. - * This is not always the case. In some circumstances (ARM notably), - * the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses. - * - * You can force the compiler to use unaligned memory access by uncommenting the line below. - * One of the below scenarios will happen : - * 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case). - * You will witness large performance improvements (+50% and up). - * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) - * The goal is to automatically detect such situations by adding your target CPU within an exception list. - * 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler - * No change will be experienced. - * 3 - Your target CPU inefficiently handle unaligned access. - * You will experience a performance loss. Comment back the line. - * 4 - Your target CPU does not handle unaligned access. - * Program will crash. - * If uncommenting results in better performance (case 1) - * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) - * An automatic detection macro will be added to match your case within future versions of the library. - */ -/* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ - - -/************************************** - CPU Feature Detection -**************************************/ -/* - * Automated efficient unaligned memory access detection - * Based on known hardware architectures - * This list will be updated thanks to feedbacks - */ -#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ - || defined(__ARM_FEATURE_UNALIGNED) \ - || defined(__i386__) || defined(__x86_64__) \ - || defined(_M_IX86) || defined(_M_X64) \ - || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ - || (defined(_M_ARM) && (_M_ARM >= 7)) -# define LZ4_UNALIGNED_ACCESS 1 -#else -# define LZ4_UNALIGNED_ACCESS 0 -#endif - -/* - * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support hardware bit count - */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/************************************** - Compiler Options -**************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - - -/************************************** - Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/************************************** - Includes -**************************************/ -#include "lz4.h" - - -/************************************** - Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** - Reading and writing into memory -**************************************/ -#define STEPSIZE sizeof(size_t) - -static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } - -static unsigned LZ4_isLittleEndian(void) -{ - const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - - -static U16 LZ4_readLE16(const void* memPtr) -{ - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) - return *(U16*)memPtr; - else - { - const BYTE* p = memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } -} - -static void LZ4_writeLE16(void* memPtr, U16 value) -{ - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) - { - *(U16*)memPtr = value; - return; - } - else - { - BYTE* p = memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } -} - - -static U16 LZ4_read16(const void* memPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - return *(U16*)memPtr; - else - { - U16 val16; - memcpy(&val16, memPtr, 2); - return val16; - } -} - -static U32 LZ4_read32(const void* memPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - return *(U32*)memPtr; - else - { - U32 val32; - memcpy(&val32, memPtr, 4); - return val32; - } -} - -static U64 LZ4_read64(const void* memPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - return *(U64*)memPtr; - else - { - U64 val64; - memcpy(&val64, memPtr, 8); - return val64; - } -} - -static size_t LZ4_read_ARCH(const void* p) -{ - if (LZ4_64bits()) - return (size_t)LZ4_read64(p); - else - return (size_t)LZ4_read32(p); -} - - -static void LZ4_copy4(void* dstPtr, const void* srcPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - { - *(U32*)dstPtr = *(U32*)srcPtr; - return; - } - memcpy(dstPtr, srcPtr, 4); -} - -static void LZ4_copy8(void* dstPtr, const void* srcPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - { - if (LZ4_64bits()) - *(U64*)dstPtr = *(U64*)srcPtr; - else - ((U32*)dstPtr)[0] = ((U32*)srcPtr)[0], - ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; - return; - } - memcpy(dstPtr, srcPtr, 8); -} - -/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - BYTE* e = dstEnd; - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } - else /* 32 bits */ - { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } - else /* Big Endian CPU */ - { - if (LZ4_64bits()) - { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } - else /* 32 bits */ - { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); -} - -static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } - -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - switch (tableType) - { - case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } -} - -static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); -} - -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ -} - -static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); -} - -static int LZ4_compress_generic( - void* ctx, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - limitedOutput_directive outputLimited, - tableType_t tableType, - dict_directive dict, - dictIssue_directive dictIssue) -{ - LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; - - const BYTE* ip = (const BYTE*) source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - dictPtr->dictSize; - const BYTE* const dictionary = dictPtr->dictionary; - const BYTE* const dictEnd = dictionary + dictPtr->dictSize; - const size_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - const int skipStrength = SKIPSTRENGTH; - U32 forwardH; - size_t refDelta=0; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - dictPtr->currentOffset; - lowLimit = (const BYTE*)source - dictPtr->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - dictPtr->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */ - if (inputSize> skipStrength; - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - ref = LZ4_getPositionOnHash(h, ctx, tableType, base); - if (dict==usingExtDict) - { - if (ref<(const BYTE*)source) - { - refDelta = dictDelta; - lowLimit = dictionary; - } - else - { - refDelta = 0; - lowLimit = (const BYTE*)source; - } - } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip)) - || (LZ4_read32(ref+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; } - - { - /* Encode Literal length */ - unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; /* Check output limit */ - if (litLength>=RUN_MASK) - { - int len = (int)litLength-RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit); - ip += MINMATCH + matchLength; - if (ip==limit) - { - unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchLength += more; - ip += more; - } - } - else - { - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit); - ip += MINMATCH + matchLength; - } - - if (matchLength>=ML_MASK) - { - if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) - return 0; /* Check output limit */ - *token += ML_MASK; - matchLength -= ML_MASK; - for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } - if (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); - - /* Test next position */ - ref = LZ4_getPosition(ip, ctx, tableType, base); - if (dict==usingExtDict) - { - if (ref<(const BYTE*)source) - { - refDelta = dictDelta; - lowLimit = dictionary; - } - else - { - refDelta = 0; - lowLimit = (const BYTE*)source; - } - } - LZ4_putPosition(ip, ctx, tableType, base); - if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1) - && (ref+MAX_DISTANCE>=ip) - && (LZ4_read32(ref+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); - if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) - return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} - -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return (0); -} - - -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ - - if (dictSize < MINMATCH) - { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-MINMATCH) - { - LZ4_putPosition(p, dict, byU32, base); - p+=3; - } - - return dict->dictSize; -} - - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) -{ - if ((LZ4_dict->currentOffset > 0x80000000) || - ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ - { - /* rescale hash table */ - U32 delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - - -FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, - int maxOutputSize, limitedOutput_directive limit) -{ - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - - /* Check overlapping input/dictionary space */ - { - const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) - { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) - { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); - else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); - else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); -} - -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); -} - - -/* Hidden debug function, to force separate dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); - - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - - return result; -} - - -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; - const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - - - -/**************************** - Decompression functions -****************************/ -/* - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is essential this generic function is really inlined, - * in order to remove useless branches during compilation optimization. - */ -FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest if dict == noDict */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - /* Local Variables */ - const BYTE* restrict ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; - const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - - /* Special cases */ - if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - - - /* Main Loop */ - while (1) - { - unsigned token; - size_t length; - const BYTE* match; - - /* get literal length */ - token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) - { - unsigned s; - do - { - s = *ip++; - length += s; - } - while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-COPYLENGTH))) - { - if (partialDecoding) - { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } - else - { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - match = cpy - LZ4_readLE16(ip); ip+=2; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) - { - unsigned s; - do - { - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - s = *ip++; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) - { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) - { - /* match can be copied as a single segment from external dictionary */ - match = dictEnd - (lowPrefix-match); - memcpy(op, match, length); - op += length; - } - else - { - /* match encompass external dictionary and current segment */ - size_t copySize = (size_t)(lowPrefix-match); - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - copySize = length - copySize; - if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ - { - BYTE* const endOfMatch = op + copySize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } - else - { - memcpy(op, lowPrefix, copySize); - op += copySize; - } - } - continue; - } - - /* copy repeated sequence */ - cpy = op + length; - if (unlikely((op-match)<8)) - { - const size_t dec64 = dec64table[op-match]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[op-match]; - LZ4_copy4(op+4, match); - op += 8; match -= dec64; - } else { LZ4_copy8(op, match); op+=8; match+=8; } - - if (unlikely(cpy>oend-12)) - { - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ - if (op < oend-8) - { - LZ4_wildCopy(op, match, oend-8); - match += (oend-8) - op; - op = oend-8; - } - while (opprefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) - { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } - else - { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) - { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } - else - { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) - { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); -} - - -/*************************************************** - Obsolete Functions -***************************************************/ -/* -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base) -{ - MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); - lz4ds->bufferStart = base; -} - -int LZ4_resetStreamState(void* state, const char* inputBuffer) -{ - if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (const char* inputBuffer) -{ - void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); - return lz4ds; -} - -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data; - - LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB); - - return (char*)(lz4ds->bufferStart + 64 KB); -} - -/* Obsolete compresson functions using User-allocated state */ - -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < (int)LZ4_64KLIMIT) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < (int)LZ4_64KLIMIT) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -#endif /* LZ4_COMMONDEFS_ONLY */ - diff --git a/lz4.h b/lz4.h deleted file mode 100644 index 22bbcb5..0000000 --- a/lz4.h +++ /dev/null @@ -1,310 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - -/* - * lz4.h provides raw compression format functions, for optimal performance and integration into programs. - * If you need to generate data using an inter-operable format (respecting the framing specification), - * please use lz4frame.h instead. -*/ - -/************************************** - Version -**************************************/ -#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */ -#define LZ4_VERSION_MINOR 4 /* for minor interface/format changes */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ -#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) -int LZ4_versionNumber (void); - -/************************************** - Tuning parameter -**************************************/ -/* - * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache - */ -#define LZ4_MEMORY_USAGE 14 - - -/************************************** - Simple Functions -**************************************/ - -int LZ4_compress (const char* source, char* dest, int sourceSize); -int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - -/* -LZ4_compress() : - Compresses 'sourceSize' bytes from 'source' into 'dest'. - Destination buffer must be already allocated, - and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() - inputSize : Max supported value is LZ4_MAX_INPUT_SIZE - return : the number of bytes written in buffer dest - or 0 if the compression fails - -LZ4_decompress_safe() : - compressedSize : is obviously the source size - maxDecompressedSize : is the size of the destination buffer, which must be already allocated. - return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) - If the destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, - and never writes outside of output buffer, nor reads outside of input buffer. - It is also protected against malicious data packets. -*/ - - -/************************************** - Advanced Functions -**************************************/ -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) - -/* -LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (output buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - - isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ -int LZ4_compressBound(int isize); - - -/* -LZ4_compress_limitedOutput() : - Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This saves time and memory on detecting non-compressible (or barely compressible) data. - This function never writes outside of provided output buffer. - - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes written in buffer 'dest' - or 0 if compression fails -*/ -int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); - - -/* -LZ4_compress_withState() : - Same compression functions, but using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and then, provide it as 'void* state' to compression functions. -*/ -int LZ4_sizeofState(void); -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); - - -/* -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - - -/* -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - -/*********************************************** - Streaming Compression Functions -***********************************************/ - -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) -/* - * LZ4_stream_t - * information structure to track an LZ4 stream. - * important : init this structure content before first use ! - */ -typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; - -/* - * LZ4_resetStream - * Use this function to init an allocated LZ4_stream_t structure - */ -void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); - -/* - * If you prefer dynamic allocation methods, - * LZ4_createStream will allocate and initialize an LZ4_stream_t structure - * LZ4_freeStream releases its memory. - */ -LZ4_stream_t* LZ4_createStream(void); -int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); - -/* - * LZ4_loadDict - * Use this function to load a static dictionary into LZ4_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) - */ -int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); - -/* - * LZ4_compress_continue - * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio - * Previous data blocks are assumed to still be present at their previous location. - */ -int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); - -/* - * LZ4_compress_limitedOutput_continue - * Same as before, but also specify a maximum target compressed size (maxOutputSize) - * If objective cannot be met, compression exits, and returns a zero. - */ -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* - * LZ4_saveDict - * If previously compressed data block is not guaranteed to remain available at its memory location - * save it into a safer place (char* safeBuffer) - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() - * Return : dictionary size in bytes, or 0 if error - * Note : any dictSize > 64 KB will be interpreted as 64KB. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); - - -/************************************************ - Streaming Decompression Functions -************************************************/ - -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; -/* - * LZ4_streamDecode_t - * information structure to track an LZ4 stream. - * init this structure content using LZ4_setStreamDecode or memset() before first use ! - * If you prefer dynamic allocation methods : - * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - * LZ4_freeStreamDecode releases its memory. - */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void); -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/* - * LZ4_setStreamDecode - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * Return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - If this condition is not possible, save the relevant part of decoded data into a safe buffer, - and indicate where is its new address using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as - a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() - They are stand-alone and don't use nor update an LZ4_streamDecode_t structure. -*/ -int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - - - -/************************************** - Obsolete Functions -**************************************/ -/* -Obsolete decompression functions -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is the same as LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe -These function prototypes are now disabled; uncomment them if you really need them. -It is highly recommended to stop using these functions and migrate to newer ones */ -/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ -/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ - - -/* Obsolete streaming functions; use new streaming interface whenever possible */ -void* LZ4_create (const char* inputBuffer); -int LZ4_sizeofStreamState(void); -int LZ4_resetStreamState(void* state, const char* inputBuffer); -char* LZ4_slideInputBuffer (void* state); - -/* Obsolete streaming decoding functions */ -int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize); -int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); - - -#if defined (__cplusplus) -} -#endif diff --git a/lz4frame.c b/lz4frame.c deleted file mode 100644 index aa46152..0000000 --- a/lz4frame.c +++ /dev/null @@ -1,1330 +0,0 @@ -/* -LZ4 auto-framing library -Copyright (C) 2011-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- LZ4 source repository : http://code.google.com/p/lz4/ -- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* LZ4F is a stand-alone API to create LZ4-compressed Frames -* fully conformant to specification v1.4.1. -* All related operations, including memory management, are handled by the library. -* */ - - -/************************************** -Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - - -/************************************** -Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#include /* memset, memcpy, memmove */ -#define MEM_INIT memset - - -/************************************** -Includes -**************************************/ -#include "lz4frame.h" -#include "lz4.h" -#include "lz4hc.h" -#include "xxhash.h" - - -/************************************** -Basic Types -**************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - -/************************************** -Constants -**************************************/ -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define LZ4F_MAGICNUMBER 0x184D2204U -#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_MAXHEADERFRAME_SIZE 7 -#define LZ4F_BLOCKSIZEID_DEFAULT max64KB - -static const U32 minHClevel = 3; - -/************************************** -Structures and local types -**************************************/ -typedef struct -{ - LZ4F_preferences_t prefs; - U32 version; - U32 cStage; - size_t maxBlockSize; - size_t maxBufferSize; - BYTE* tmpBuff; - BYTE* tmpIn; - size_t tmpInSize; - XXH32_state_t xxh; - void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ -} LZ4F_cctx_internal_t; - -typedef struct -{ - LZ4F_frameInfo_t frameInfo; - unsigned version; - unsigned dStage; - size_t maxBlockSize; - size_t maxBufferSize; - const BYTE* srcExpect; - BYTE* tmpIn; - size_t tmpInSize; - size_t tmpInTarget; - BYTE* tmpOutBuffer; - BYTE* dict; - size_t dictSize; - BYTE* tmpOut; - size_t tmpOutSize; - size_t tmpOutStart; - XXH32_state_t xxh; - BYTE header[8]; -} LZ4F_dctx_internal_t; - - -/************************************** -Macros -**************************************/ - - -/************************************** -Error management -**************************************/ -#define LZ4F_GENERATE_STRING(STRING) #STRING, -static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; - - -int LZ4F_isError(LZ4F_errorCode_t code) -{ - return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); -} - -const char* LZ4F_getErrorName(LZ4F_errorCode_t code) -{ - static const char* codeError = "Unspecified error code"; - if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; - return codeError; -} - - -/************************************** -Private functions -**************************************/ -static size_t LZ4F_getBlockSize(unsigned blockSizeID) -{ - static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; - - if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - blockSizeID -= 4; - if (blockSizeID > 3) return (size_t)-ERROR_maxBlockSize_invalid; - return blockSizes[blockSizeID]; -} - - -/* unoptimized version; solves endianess & alignment issues */ -static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) -{ - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - -static U32 LZ4F_readLE32 (const BYTE* srcPtr) -{ - U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); - return value32; -} - - -static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) -{ - U32 xxh = XXH32(header, (U32)length, 0); - return (BYTE)(xxh >> 8); -} - - -/************************************** -Simple compression functions -**************************************/ -size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_preferences_t prefs = { 0 }; - size_t headerSize; - size_t streamSize; - - if (preferencesPtr!=NULL) prefs = *preferencesPtr; - { - blockSizeID_t proposedBSID = max64KB; - size_t maxBlockSize = 64 KB; - while (prefs.frameInfo.blockSizeID > proposedBSID) - { - if (srcSize <= maxBlockSize) - { - prefs.frameInfo.blockSizeID = proposedBSID; - break; - } - proposedBSID++; - maxBlockSize <<= 2; - } - } - prefs.autoFlush = 1; - - headerSize = 7; /* basic header size (no option) including magic number */ - streamSize = LZ4F_compressBound(srcSize, &prefs); - - return headerSize + streamSize; -} - - -/* LZ4F_compressFrame() -* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() -* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) -* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. -* The result of the function is the number of bytes written into dstBuffer. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ - LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressOptions_t options = { 0 }; - LZ4F_errorCode_t errorCode; - BYTE* const dstStart = (BYTE*) dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* const dstEnd = dstStart + dstMaxSize; - - - cctxI.version = LZ4F_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ - - if (preferencesPtr!=NULL) prefs = *preferencesPtr; - { - blockSizeID_t proposedBSID = max64KB; - size_t maxBlockSize = 64 KB; - while (prefs.frameInfo.blockSizeID > proposedBSID) - { - if (srcSize <= maxBlockSize) - { - prefs.frameInfo.blockSizeID = proposedBSID; - break; - } - proposedBSID++; - maxBlockSize <<= 2; - } - } - prefs.autoFlush = 1; - if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */ - - options.stableSrc = 1; - - if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) - return (size_t)-ERROR_dstMaxSize_tooSmall; - - errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; /* header size */ - - dstMaxSize -= errorCode; - errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options); - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; - - errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; - - FREEMEM(cctxI.lz4CtxPtr); - - return (dstPtr - dstStart); -} - - -/*********************************** -* Advanced compression functions -* *********************************/ - -/* LZ4F_createCompressionContext() : -* The first thing to do is to create a compressionContext object, which will be used in all compression operations. -* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. -* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. -* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. -* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. -* Object can release its memory using LZ4F_freeCompressionContext(); -*/ -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) -{ - LZ4F_cctx_internal_t* cctxPtr; - - cctxPtr = (LZ4F_cctx_internal_t*)ALLOCATOR(sizeof(LZ4F_cctx_internal_t)); - if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-ERROR_allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : write header */ - - *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; - - return OK_NoError; -} - - -LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) -{ - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; - - FREEMEM(cctxPtr->lz4CtxPtr); - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LZ4F_compressionContext); - - return OK_NoError; -} - - -/* LZ4F_compressBegin() : -* will write the frame header into dstBuffer. -* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. -* The result of the function is the number of bytes written into dstBuffer for the header -* or an error code (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) -{ - LZ4F_preferences_t prefNull = { 0 }; - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* headerStart; - size_t requiredBuffSize; - - if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; - if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; - if (preferencesPtr == NULL) preferencesPtr = &prefNull; - cctxPtr->prefs = *preferencesPtr; - - /* ctx Management */ - { - U32 targetCtxLevel = cctxPtr->prefs.compressionLevellz4CtxLevel < targetCtxLevel) - { - FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevellz4CtxPtr = (void*)LZ4_createStream(); - else - cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); - cctxPtr->lz4CtxLevel = targetCtxLevel; - } - } - - /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; - cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 128 KB); - if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 64 KB; /* just needs dict */ - - if (cctxPtr->maxBufferSize < requiredBuffSize) - { - cctxPtr->maxBufferSize = requiredBuffSize; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return (size_t)-ERROR_allocation_failed; - } - cctxPtr->tmpIn = cctxPtr->tmpBuff; - cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevellz4CtxPtr)); - else - LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); - - /* Magic Number */ - LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); - dstPtr += 4; - headerStart = dstPtr; - - /* FLG Byte */ - *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ - /* BD Byte */ - *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - /* CRC Byte */ - *dstPtr++ = LZ4F_headerChecksum(headerStart, 2); - - cctxPtr->cStage = 1; /* header written, wait for data block */ - - return (dstPtr - dstStart); -} - - -/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. -* The LZ4F_frameInfo_t structure is optional : -* you can provide NULL as argument, all preferences will then be set to default. -* */ -size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) -{ - const LZ4F_preferences_t prefsNull = { 0 }; - const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LZ4F_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; - - return result; -} - - -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); - -static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) -{ - /* compress one block */ - BYTE* cSizePtr = (BYTE*)dst; - U32 cSize; - cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); - LZ4F_writeLE32(cSizePtr, cSize); - if (cSize == 0) /* compression failed */ - { - cSize = (U32)srcSize; - LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); - memcpy(cSizePtr+4, src, srcSize); - } - return cSize + 4; -} - - -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) -{ - (void) level; - return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); -} - -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) -{ - (void) level; - return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); -} - -static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) -{ - (void) level; - return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); -} - -static compressFunc_t LZ4F_selectCompression(blockMode_t blockMode, U32 level) -{ - if (level < minHClevel) - { - if (blockMode == blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; - return LZ4F_localLZ4_compress_limitedOutput_continue; - } - if (blockMode == blockIndependent) return LZ4_compressHC2_limitedOutput_withStateHC; - return LZ4F_localLZ4_compressHC_limitedOutput_continue; -} - -static int LZ4F_localSaveDict(LZ4F_cctx_internal_t* cctxPtr) -{ - if (cctxPtr->prefs.compressionLevel < minHClevel) - return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); - return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); -} - -typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; - -/* LZ4F_compressUpdate() -* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -*/ -size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) -{ - LZ4F_compressOptions_t cOptionsNull = { 0 }; - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; - size_t blockSize = cctxPtr->maxBlockSize; - const BYTE* srcPtr = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcPtr + srcSize; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - LZ4F_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t compress; - - - if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; - if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - - /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - - /* complete tmp buffer */ - if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ - { - size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; - if (sizeToCopy > srcSize) - { - /* add src to tmpIn buffer */ - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); - srcPtr = srcEnd; - cctxPtr->tmpInSize += srcSize; - /* still needs some CRC */ - } - else - { - /* complete tmpIn block and then compress it */ - lastBlockCompressed = fromTmpBuffer; - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); - srcPtr += sizeToCopy; - - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - - if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize; - cctxPtr->tmpInSize = 0; - } - } - - while ((size_t)(srcEnd - srcPtr) >= blockSize) - { - /* compress full block */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - srcPtr += blockSize; - } - - if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) - { - /* compress remaining input < blockSize */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - srcPtr = srcEnd; - } - - /* preserve dictionary if necessary */ - if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer)) - { - if (compressOptionsPtr->stableSrc) - { - cctxPtr->tmpIn = cctxPtr->tmpBuff; - } - else - { - int realDictSize = LZ4F_localSaveDict(cctxPtr); - if (realDictSize==0) return (size_t)-ERROR_GENERIC; - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - } - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ - && !(cctxPtr->prefs.autoFlush)) - { - LZ4F_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; - } - - /* some input data left, necessarily < blockSize */ - if (srcPtr < srcEnd) - { - /* fill tmp buffer */ - size_t sizeToCopy = srcEnd - srcPtr; - memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); - cctxPtr->tmpInSize = sizeToCopy; - } - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize); - - return dstPtr - dstStart; -} - - -/* LZ4F_flush() -* Should you need to create compressed data immediately, without waiting for a block to be filled, -* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. -* The result of the function is the number of bytes written into dstBuffer -* (it can be zero, this means there was no data left within compressionContext) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -*/ -size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) -{ - LZ4F_compressOptions_t cOptionsNull = { 0 }; - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - compressFunc_t compress; - - - if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; - if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - (void)compressOptionsPtr; /* not yet useful */ - - /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - - /* compress tmp buffer */ - dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; - cctxPtr->tmpInSize = 0; - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */ - { - LZ4F_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + 64 KB; - } - - return dstPtr - dstStart; -} - - -/* LZ4F_compressEnd() -* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). -* It will flush whatever data remained within compressionContext (like LZ4_flush()) -* but also properly finalize the frame, with an endMark and a checksum. -* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) -* The function outputs an error code if it fails (can be tested using LZ4F_isError()) -* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. -* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. -*/ -size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) -{ - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - size_t errorCode; - - errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); - if (LZ4F_isError(errorCode)) return errorCode; - dstPtr += errorCode; - - LZ4F_writeLE32(dstPtr, 0); - dstPtr+=4; /* endMark */ - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) - { - U32 xxh = XXH32_digest(&(cctxPtr->xxh)); - LZ4F_writeLE32(dstPtr, xxh); - dstPtr+=4; /* content Checksum */ - } - - cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ - - return dstPtr - dstStart; -} - - -/*********************************** -* Decompression functions -* *********************************/ - -/* Resource management */ - -/* LZ4F_createDecompressionContext() : -* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. -* This is achieved using LZ4F_createDecompressionContext(). -* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. -* If the result LZ4F_errorCode_t is not zero, there was an error during context creation. -* Object can release its memory using LZ4F_freeDecompressionContext(); -*/ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) -{ - LZ4F_dctx_internal_t* dctxPtr; - - dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); - if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; - - dctxPtr->version = versionNumber; - *LZ4F_decompressionContextPtr = (LZ4F_compressionContext_t)dctxPtr; - return OK_NoError; -} - -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext) -{ - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext; - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - FREEMEM(dctxPtr); - return OK_NoError; -} - - -/* Decompression */ - -static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize) -{ - BYTE FLG, BD, HC; - unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; - size_t bufferNeeded; - - /* need to decode header to get frameInfo */ - if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ - - /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC; - srcPtr += 4; - - /* Flags */ - FLG = srcPtr[0]; - version = (FLG>>6)&_2BITS; - blockMode = (FLG>>5) & _1BIT; - blockChecksumFlag = (FLG>>4) & _1BIT; - contentSizeFlag = (FLG>>3) & _1BIT; - contentChecksumFlag = (FLG>>2) & _1BIT; - dictFlag = (FLG>>0) & _1BIT; - BD = srcPtr[1]; - blockSizeID = (BD>>4) & _3BITS; - - /* check */ - HC = LZ4F_headerChecksum(srcPtr, 2); - if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ - - /* validate */ - if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ - - /* save */ - dctxPtr->frameInfo.blockMode = blockMode; - dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = blockSizeID; - dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); - - /* init */ - if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); - - /* alloc */ - bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==blockLinked) * 128 KB); - if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ - { - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - dctxPtr->maxBufferSize = bufferNeeded; - dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; - dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); - if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; - } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = 0; - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = 0; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; - dctxPtr->tmpOutStart = 0; - dctxPtr->tmpOutSize = 0; - - return 7; -} - - -typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, - dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix -} dStage_t; - - -/* LZ4F_getFrameInfo() -* This function decodes frame header information, such as blockSize. -* It is optional : you could start by calling directly LZ4F_decompress() instead. -* The objective is to extract header information without starting decompression, typically for allocation purposes. -* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. -* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) -* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, -* or an error code which can be tested using LZ4F_isError(). -*/ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) -{ - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; - - if (dctxPtr->dStage == dstage_getHeader) - { - LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr); - if (LZ4F_isError(errorCode)) return errorCode; - *srcSizePtr = errorCode; - *frameInfoPtr = dctxPtr->frameInfo; - dctxPtr->srcExpect = NULL; - dctxPtr->dStage = dstage_getCBlockSize; - return 4; - } - - /* frameInfo already decoded */ - *srcSizePtr = 0; - *frameInfoPtr = dctxPtr->frameInfo; - return 0; -} - - -static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) -{ - (void)dictStart; - (void)dictSize; - return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); -} - - - -static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) -{ - if (dctxPtr->dictSize==0) - dctxPtr->dict = (BYTE*)dstPtr; /* priority to dictionary continuity */ - - if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ - { - dctxPtr->dictSize += dstSize; - return; - } - - if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ - { - dctxPtr->dict = (BYTE*)dstPtr0; - dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; - return; - } - - if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) - { - /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ - dctxPtr->dictSize += dstSize; - return; - } - - if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - { -#if 0 - size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize; - return; - -#else - - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; - return; -#endif - } - - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ - { - if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ - { - size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - dctxPtr->dictSize = preserveSize; - } - memcpy(dctxPtr->dict + dctxPtr->dictSize, dstPtr, dstSize); - dctxPtr->dictSize += dstSize; - return; - } - - /* join dict & dest into tmp */ - { - size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dstSize; - } -} - - - -/* LZ4F_decompress() -* Call this function repetitively to regenerate data compressed within srcBuffer. -* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. -* -* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). -* -* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. -* You will have to call it again, continuing from where it stopped. -* -* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. -* Basically, it's the size of the current (or remaining) compressed block + header of next block. -* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. -* Note that this is just a hint, you can always provide any srcSize you want. -* When a frame is fully decoded, the function result will be 0. -* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). -*/ -size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* decompressOptionsPtr) -{ - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; - static const LZ4F_decompressOptions_t optionsNull = { 0 }; - const BYTE* const srcStart = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcStart + *srcSizePtr; - const BYTE* srcPtr = srcStart; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* const dstEnd = dstStart + *dstSizePtr; - BYTE* dstPtr = dstStart; - const BYTE* selectedIn=NULL; - unsigned doAnotherStage = 1; - size_t nextSrcSizeHint = 1; - - - if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; - *srcSizePtr = 0; - *dstSizePtr = 0; - - /* expect to continue decoding src buffer where it left previously */ - if (dctxPtr->srcExpect != NULL) - { - if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_GENERIC; - } - - /* programmed as a state machine */ - - while (doAnotherStage) - { - - switch(dctxPtr->dStage) - { - - case dstage_getHeader: - { - if (srcEnd-srcPtr >= 7) - { - selectedIn = srcPtr; - srcPtr += 7; - dctxPtr->dStage = dstage_decodeHeader; - break; - } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeHeader; - break; - } - - case dstage_storeHeader: - { - size_t sizeToCopy = 7 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < 7) - { - nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; - doAnotherStage = 0; /* no enough src, wait to get some more */ - break; - } - selectedIn = dctxPtr->header; - dctxPtr->dStage = dstage_decodeHeader; - break; - } - - case dstage_decodeHeader: - { - LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7); - if (LZ4F_isError(errorCode)) return errorCode; - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - - case dstage_getCBlockSize: - { - if ((srcEnd - srcPtr) >= 4) - { - selectedIn = srcPtr; - srcPtr += 4; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; - } - /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; - break; - } - - case dstage_storeCBlockSize: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */ - { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; - break; - } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; - } - - case dstage_decodeCBlockSize: - { - size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; - if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ - { - dctxPtr->dStage = dstage_getSuffix; - break; - } - if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */ - dctxPtr->tmpInTarget = nextCBlockSize; - if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) - { - dctxPtr->dStage = dstage_copyDirect; - break; - } - dctxPtr->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) - { - nextSrcSizeHint = nextCBlockSize + 4; - doAnotherStage = 0; - } - break; - } - - case dstage_copyDirect: /* uncompressed block */ - { - size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ - if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; - memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, (U32)sizeToCopy); - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); - - srcPtr += sizeToCopy; - dstPtr += sizeToCopy; - if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ - { - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + 4; - doAnotherStage = 0; - break; - } - - case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ - { - if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) - { - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlock; - break; - } - selectedIn = srcPtr; - srcPtr += dctxPtr->tmpInTarget; - dctxPtr->dStage = dstage_decodeCBlock; - break; - } - - case dstage_storeCBlock: - { - size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ - { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; - doAnotherStage=0; - break; - } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlock; - break; - } - - case dstage_decodeCBlock: - { - if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctxPtr->dStage = dstage_decodeCBlock_intoTmp; - else - dctxPtr->dStage = dstage_decodeCBlock_intoDst; - break; - } - - case dstage_decodeCBlock_intoDst: - { - int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); - - dstPtr += decodedSize; - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - - case dstage_decodeCBlock_intoTmp: - { - /* not enough place into dst : decode into tmpOut */ - int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == blockLinked) - decoder = LZ4_decompress_safe_usingDict; - else - decoder = LZ4F_decompress_safe; - - /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == blockLinked) - { - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) - { - if (dctxPtr->dictSize > 128 KB) - { - memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); - dctxPtr->dictSize = 64 KB; - } - dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize; - } - else /* dict not within tmp */ - { - size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; - } - } - - /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - dctxPtr->tmpOutSize = decodedSize; - dctxPtr->tmpOutStart = 0; - dctxPtr->dStage = dstage_flushOut; - break; - } - - case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { - size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; - if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; - memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) - LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); - - dctxPtr->tmpOutStart += sizeToCopy; - dstPtr += sizeToCopy; - - /* end of flush ? */ - if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) - { - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - nextSrcSizeHint = 4; - doAnotherStage = 0; /* still some data to flush */ - break; - } - - case dstage_getSuffix: - { - size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (suffixSize == 0) /* frame completed */ - { - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } - if ((srcEnd - srcPtr) >= 4) /* CRC present */ - { - selectedIn = srcPtr; - srcPtr += 4; - dctxPtr->dStage = dstage_checkSuffix; - break; - } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - break; - } - - case dstage_storeSuffix: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ - { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; - break; - } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_checkSuffix; - break; - } - - case dstage_checkSuffix: - { - U32 readCRC = LZ4F_readLE32(selectedIn); - U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } - } - } - - /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) - { - if (dctxPtr->dStage == dstage_flushOut) - { - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = 64 KB - dctxPtr->tmpOutSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; - } - else - { - size_t newDictSize = dctxPtr->dictSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; - if ((newDictSize) > 64 KB) newDictSize = 64 KB; - - memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = newDictSize; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; - } - } - - if (srcPtrsrcExpect = srcPtr; - else - dctxPtr->srcExpect = NULL; - *srcSizePtr = (srcPtr - srcStart); - *dstSizePtr = (dstPtr - dstStart); - return nextSrcSizeHint; -} diff --git a/lz4frame.h b/lz4frame.h deleted file mode 100644 index a4f5e36..0000000 --- a/lz4frame.h +++ /dev/null @@ -1,261 +0,0 @@ -/* - LZ4 auto-framing library - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* LZ4F is a stand-alone API to create LZ4-compressed frames - * fully conformant to specification v1.4.1. - * All related operations, including memory management, are handled by the library. - * You don't need lz4.h when using lz4frame.h. - * */ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - -/**************************************** - Note : experimental API. - Not yet integrated within lz4 library. -****************************************/ - -/************************************** - Includes -**************************************/ -#include /* size_t */ - - -/************************************** - Error management -**************************************/ -typedef size_t LZ4F_errorCode_t; -#define LZ4F_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ - ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_checksum_invalid) \ - ITEM(ERROR_maxCode) - -#define LZ4F_GENERATE_ENUM(ENUM) ENUM, -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ - -int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ -const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */ - - -/************************************** - Framing compression functions -**************************************/ - -typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} blockSizeID_t; -typedef enum { blockLinked=0, blockIndependent} blockMode_t; -typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; - -typedef struct { - blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - unsigned reserved[5]; -} LZ4F_frameInfo_t; - -typedef struct { - LZ4F_frameInfo_t frameInfo; - unsigned compressionLevel; /* Not yet supported : only fast compression for the time being */ - unsigned autoFlush; /* 1 == always flush; reduce need for tmp buffer */ - unsigned reserved[4]; -} LZ4F_preferences_t; - - - -/*********************************** - * Simple compression function - * *********************************/ -size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - -size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -/* LZ4F_compressFrame() - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() - * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - */ - - - -/********************************** - * Advanced compression functions - * *********************************/ - -typedef void* LZ4F_compressionContext_t; - -typedef struct { - unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ - unsigned reserved[3]; -} LZ4F_compressOptions_t; - -/* Resource Management */ - -#define LZ4F_VERSION 100 -LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version); -LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext); -/* LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. - * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object. - * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); - */ - - -/* Compression */ - -size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr); -/* LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes. - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. - * The result of the function is the number of bytes written into dstBuffer for the header - * or an error code (can be tested using LZ4F_isError()) - */ - -size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); -/* LZ4F_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * preferencesPtr is optional : you can provide NULL as argument, all preferences will then be set to default. - */ - -size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); -/* LZ4F_compressUpdate() - * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) - * You can get the minimum value of dstMaxSize by using LZ4F_compressBound() - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - */ - -size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); -/* LZ4F_flush() - * Should you need to create compressed data immediately, without waiting for a block to be filled, - * you can call LZ4_flush(), which will immediately compress any remaining data buffered within compressionContext. - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within compressionContext) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - */ - -size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr); -/* LZ4F_compressEnd() - * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). - * It will flush whatever data remained within compressionContext (like LZ4_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) - * The function outputs an error code if it fails (can be tested using LZ4F_isError()) - * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * compressionContext can then be used again, starting with LZ4F_compressBegin(). - */ - - -/*********************************** - * Decompression functions - * *********************************/ - -typedef void* LZ4F_decompressionContext_t; - -typedef struct { - unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ - unsigned reserved[3]; -} LZ4F_decompressOptions_t; - -/* Resource management */ - -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned version); -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext); -/* LZ4F_createDecompressionContext() : - * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. - * This is achieved using LZ4F_createDecompressionContext(). - * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. - * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. - * Object can release its memory using LZ4F_freeDecompressionContext(); - */ - -/* Decompression */ - -size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); -/* LZ4F_getFrameInfo() - * This function decodes frame header information, such as blockSize. - * It is optional : you could start by calling directly LZ4F_decompress() instead. - * The objective is to extract header information without starting decompression, typically for allocation purposes. - * LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. - * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, - * or an error code which can be tested using LZ4F_isError(). - */ - -size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr); -/* LZ4F_decompress() - * Call this function repetitively to regenerate data compressed within srcBuffer. - * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. - * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). - * - * The number of bytes effectively used from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. - * This typically happens when dstBuffer is not large enough to contain all decoded data. - * LZ4F_decompress() will have to be called again, starting from where it stopped (srcBuffer + *srcSizePtr) - * The function will check this condition, and refuse to continue if it is not respected. - * dstBuffer is supposed to be flushed between calls to the function, since its content will be rewritten. - * Different dst arguments can be used between each calls. - * - * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. - * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, you can always provide any srcSize you want. - * When a frame is fully decoded, the function result will be 0. - * If decompression failed, function result is an error code which can be tested using LZ4F_isError(). - */ - - - -#if defined (__cplusplus) -} -#endif diff --git a/lz4hc.c b/lz4hc.c deleted file mode 100644 index 89f0db0..0000000 --- a/lz4hc.c +++ /dev/null @@ -1,768 +0,0 @@ -/* -LZ4 HC - High Compression Mode of LZ4 -Copyright (C) 2011-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html -- LZ4 source repository : http://code.google.com/p/lz4/ -*/ - - - -/************************************** - Tuning Parameter -**************************************/ -#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 - - -/************************************** - Includes -**************************************/ -#include "lz4.h" -#include "lz4hc.h" - - -/************************************** - Local Compiler Options -**************************************/ -#if defined(__GNUC__) -# pragma GCC diagnostic ignored "-Wunused-function" -#endif - -#if defined (__clang__) -# pragma clang diagnostic ignored "-Wunused-function" -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# pragma warning(disable : 4201) /* disable: C4201: unnamed struct/union*/ -#endif - - -/************************************** - Common LZ4 definition -**************************************/ -#define LZ4_COMMONDEFS_ONLY -#include "lz4.c" - - -/************************************** - Local Constants -**************************************/ -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) -#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] -#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) - -static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } - - - -/************************************** - HC Compression -**************************************/ -static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* base) -{ - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); - MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = 64 KB; - hc4->base = base - 64 KB; - hc4->inputBuffer = base; - hc4->end = base; - hc4->dictBase = base - 64 KB; - hc4->dictLimit = 64 KB; - hc4->lowLimit = 64 KB; -} - - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) -{ - U16* chainTable = hc4->chainTable; - U32* HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const U32 target = (U32)(ip - base); - U32 idx = hc4->nextToUpdate; - - while(idx < target) - { - U32 h = LZ4HC_hashPtr(base+idx); - size_t delta = idx - HashTable[h]; - if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - chainTable[idx & 0xFFFF] = (U16)delta; - HashTable[h] = idx; - idx++; - } - - hc4->nextToUpdate = target; -} - - -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Note : need to handle risk of index overflow - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos, - const int maxNbAttempts) -{ - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const BYTE* const dictBase = hc4->dictBase; - const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - U32 matchIndex; - const BYTE* match; - int nbAttempts=maxNbAttempts; - size_t ml=0; - - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - - while ((matchIndex>=lowLimit) && (nbAttempts)) - { - nbAttempts--; - if (matchIndex >= dictLimit) - { - match = base + matchIndex; - if (*(match+ml) == *(ip+ml) - && (LZ4_read32(match) == LZ4_read32(ip))) - { - size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } - else - { - match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { - size_t mlt; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iLimit) vLimit = iLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } // virtual matchpos - } - } - matchIndex -= chainTable[matchIndex & 0xFFFF]; - } - - return (int)ml; -} - - -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_Data_Structure* hc4, - const BYTE* ip, - const BYTE* iLowLimit, - const BYTE* iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos, - const int maxNbAttempts) -{ - U16* const chainTable = hc4->chainTable; - U32* const HashTable = hc4->hashTable; - const BYTE* const base = hc4->base; - const U32 dictLimit = hc4->dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); - const BYTE* const dictBase = hc4->dictBase; - const BYTE* match; - U32 matchIndex; - int nbAttempts = maxNbAttempts; - int delta = (int)(ip-iLowLimit); - - - /* First Match */ - LZ4HC_Insert(hc4, ip); - matchIndex = HashTable[LZ4HC_hashPtr(ip)]; - - while ((matchIndex>=lowLimit) && (nbAttempts)) - { - nbAttempts--; - if (matchIndex >= dictLimit) - { - match = base + matchIndex; - if (*(iLowLimit + longest) == *(match - delta + longest)) - if (LZ4_read32(match) == LZ4_read32(ip)) - { - const BYTE* startt = ip; - const BYTE* tmpMatch = match; - const BYTE* const matchEnd = ip + MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - - while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} - - if ((matchEnd-startt) > longest) - { - longest = (int)(matchEnd-startt); - *matchpos = tmpMatch; - *startpos = startt; - } - } - } - else - { - match = dictBase + matchIndex; - if (LZ4_read32(match) == LZ4_read32(ip)) - { - size_t mlt; - int back=0; - const BYTE* vLimit = ip + (dictLimit - matchIndex); - if (vLimit > iHighLimit) vLimit = iHighLimit; - mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) - mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - matchIndex -= chainTable[matchIndex & 0xFFFF]; - } - - return longest; -} - - -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; - -//static unsigned debug = 0; - -FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, - int matchLength, - const BYTE* const match, - limitedOutput_directive limitedOutputBuffer, - BYTE* oend) -{ - int length; - BYTE* token; - - //if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); // debug - - /* Encode Literal length */ - length = (int)(*ip - *anchor); - token = (*op)++; - if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ - if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } - else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ - if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } - else *token += (BYTE)(length); - - /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; - - return 0; -} - - -#define MAX_COMPRESSION_LEVEL 16 -static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - int compressionLevel, - limitedOutput_directive limit - ) -{ - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - - unsigned maxNbAttempts; - int ml, ml2, ml3, ml0; - const BYTE* ref=NULL; - const BYTE* start2=NULL; - const BYTE* ref2=NULL; - const BYTE* start3=NULL; - const BYTE* ref3=NULL; - const BYTE* start0; - const BYTE* ref0; - - - /* init */ - if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; - if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; - maxNbAttempts = 1 << compressionLevel; - ctx->end += inputSize; - - ip++; - - /* Main Loop */ - while (ip < mflimit) - { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); - else ml2 = ml; - - if (ml2 == ml) /* No better match */ - { - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - continue; - } - - if (start0 < ip) - { - if (start2 < ip + ml0) /* empirical */ - { - ip = start0; - ref = ref0; - ml = ml0; - } - } - - /* Here, start0==ip */ - if ((start2 - ip) < 3) /* First Match too small : removed */ - { - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ - if ((start2 - ip) < OPTIMAL_ML) - { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = new_ml - (int)(start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - - if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); - else ml3 = ml2; - - if (ml3 == ml2) /* No better match : 2 sequences to encode */ - { - /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); - /* Now, encode 2 sequences */ - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - ip = start2; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; - continue; - } - - if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ - { - if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - { - if (start2 < ip+ml) - { - int correction = (int)(ip+ml - start2); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) - { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) - { - if ((start2 - ip) < (int)ML_MASK) - { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = ml - (int)(start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - else - { - ml = (int)(start2 - ip); - } - } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - } - - /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<base = NULL; - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; -} - -int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) -{ - LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; - if (dictSize > 64 KB) - { - dictionary += dictSize - 64 KB; - dictSize = 64 KB; - } - LZ4HC_init (streamPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (streamPtr, (const BYTE*)dictionary +(dictSize-3)); - streamPtr->end = (const BYTE*)dictionary + dictSize; - return dictSize; -} - - -/* compression */ - -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, - const char* source, char* dest, - int inputSize, int maxOutputSize, limitedOutput_directive limit) -{ - /* auto-init if forgotten */ - if (dsPtr->base == NULL) - LZ4HC_init (dsPtr, (const BYTE*) source); - - /* Check overflow */ - if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB) - { - size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit; - if (dictSize > 64 KB) dictSize = 64 KB; - - LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize); - } - - /* Check if blocks follow each other */ - if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); - - /* Check overlapping input/dictionary space */ - { - const BYTE* sourceEnd = (const BYTE*) source + inputSize; - const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; - const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) - { - if (sourceEnd > dictEnd) sourceEnd = dictEnd; - dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); - if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; - } - } - - return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); -} - -int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) -{ - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); -} - -int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); -} - - -/* dictionary saving */ - -int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) -{ - LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; - int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); - if (dictSize > 64 KB) dictSize = 64 KB; - if (dictSize < 4) dictSize = 0; - if (dictSize > prefixSize) dictSize = prefixSize; - memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); - //LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); - { - U32 endIndex = (U32)(streamPtr->end - streamPtr->base); - streamPtr->end = (const BYTE*)safeBuffer + dictSize; - streamPtr->base = streamPtr->end - endIndex; - streamPtr->dictLimit = endIndex - dictSize; - streamPtr->lowLimit = endIndex - dictSize; - } - return dictSize; -} - - -/*********************************** - * Deprecated Functions - ***********************************/ -int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } - -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) -{ - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - return 0; -} - -void* LZ4_createHC (const char* inputBuffer) -{ - void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); - LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - return hc4; -} - -int LZ4_freeHC (void* LZ4HC_Data) -{ - FREEMEM(LZ4HC_Data); - return (0); -} - -/* -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) -{ -return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); -} -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) -{ -return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); -} -*/ - -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) -{ - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); -} - -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) -{ - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); -} - -char* LZ4_slideInputBufferHC(void* LZ4HC_Data) -{ - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; - - if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ - - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); -} diff --git a/lz4hc.h b/lz4hc.h deleted file mode 100644 index 26ba6d3..0000000 --- a/lz4hc.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ -#pragma once - - -#if defined (__cplusplus) -extern "C" { -#endif - - -int LZ4_compressHC (const char* source, char* dest, int inputSize); -/* -LZ4_compressHC : - return : the number of bytes in compressed buffer dest - or 0 if compression fails. - note : destination buffer must be already allocated. - To avoid any problem, size it to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") -*/ - -int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); -/* -LZ4_compress_limitedOutput() : - Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This function never writes outside of provided output buffer. - - inputSize : Max supported value is 1 GB - maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) - return : the number of output bytes written in buffer 'dest' - or 0 if compression fails. -*/ - - -int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -/* - Same functions as above, but with programmable 'compressionLevel'. - Recommended values are between 4 and 9, although any value between 0 and 16 will work. - 'compressionLevel'==0 means use default 'compressionLevel' value. - Values above 16 behave the same as 16. - Equivalent variants exist for all other compression functions below. -*/ - -/* Note : - Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) -*/ - - -/************************************** - Using an external allocation -**************************************/ -int LZ4_sizeofStateHC(void); -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); - -int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); - -/* -These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. -To know how much memory must be allocated for the compression tables, use : -int LZ4_sizeofStateHC(); - -Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). - -The allocated memory can be provided to the compression functions using 'void* state' parameter. -LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. -They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap). -*/ - - - -/************************************** - Experimental Streaming Functions -**************************************/ -#define LZ4_STREAMHCSIZE_U64 32774 -#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U64 * sizeof(unsigned long long)) -typedef struct { unsigned long long table[LZ4_STREAMHCSIZE_U64]; } LZ4_streamHC_t; - -/* -This structure allows static allocation of LZ4 HC streaming state. -State must then be initialized using LZ4_resetStreamHC() before first use. - -If you prefer dynamic allocation, please refer to functions below. -*/ - -LZ4_streamHC_t* LZ4_createStreamHC(void); -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); - -/* -These functions create and release memory for LZ4 HC streaming state. -Newly created states are already initialized. -Existing state space can be re-used anytime using LZ4_resetStreamHC(). -*/ - -void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); -int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize); - -int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize); - -/* -These functions compress data in successive blocks of any size, using previous blocks as dictionary. -One key assumption is that each previous block will remain read-accessible while compressing next block. - -Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). -A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). - -Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block. -They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. -Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - -If, for any reason, previous data block can't be preserved in memory during next compression block, -you must save it to a safer memory space, -using LZ4_saveDictHC(). -*/ - - - -/************************************** - Deprecated Streaming Functions -**************************************/ -/* Note : these streaming functions still follows the older model */ -void* LZ4_createHC (const char* inputBuffer); -//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); -//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); -char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -int LZ4_freeHC (void* LZ4HC_Data); - -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); - -/* -These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. -In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : - -void* LZ4_createHC (const char* inputBuffer); -The result of the function is the (void*) pointer on the LZ4HC Data Structure. -This pointer will be needed in all other functions. -If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. -The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. -To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). -Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), -but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. -If next block does not begin immediately after the previous one, the compression will fail (return 0). - -When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : -char* LZ4_slideInputBufferHC(void* LZ4HC_Data); -must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. -Note that, for this function to work properly, minimum size of an input buffer must be 192KB. -==> The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. -*/ - -int LZ4_sizeofStreamStateHC(void); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); - -/* -These functions achieve the same result as : -void* LZ4_createHC (const char* inputBuffer); - -They are provided here to allow the user program to allocate memory using its own routines. - -To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); -Note also that space must be aligned for pointers (32 or 64 bits). - -Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned for pointers (32 or 64 bits), and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamStateHC() must be 0 is OK. -Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). -*/ - - -#if defined (__cplusplus) -} -#endif diff --git a/programs/Makefile b/programs/Makefile index 9001e2c..96de0e3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -37,11 +37,11 @@ PREFIX ?= /usr CC := $(CC) CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DLZ4_VERSION=\"$(RELEASE)\" -FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man/man1 -LZ4DIR=.. +LZ4DIR=../lib TEST_FILES = COPYING TEST_TARGETS=test-native diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 8a8fc27..e69a84a 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -288,7 +288,7 @@ int main(int argc, char** argv) legacy_format=0, forceStdout=0, forceCompress=0, - pause=0; + main_pause=0; char* input_filename=0; char* output_filename=0; char* dynNameSpace=0; @@ -424,7 +424,7 @@ int main(int argc, char** argv) break; // Pause at the end (hidden option) - case 'p': pause=1; BMK_SetPause(); break; + case 'p': main_pause=1; BMK_SetPause(); break; EXTENDED_ARGUMENTS; @@ -517,7 +517,7 @@ int main(int argc, char** argv) } } - if (pause) waitEnter(); + if (main_pause) waitEnter(); free(dynNameSpace); return 0; } diff --git a/xxhash.c b/xxhash.c deleted file mode 100644 index 24a64b5..0000000 --- a/xxhash.c +++ /dev/null @@ -1,928 +0,0 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- public discussion board : https://groups.google.com/forum/#!forum/lz4c -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory routines -// for malloc(), free() -#include -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -// for memcpy() -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S -{ - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static inline U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static inline U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch -static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); - else - return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) - { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_get32bits(p) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_get32bits(p) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_get32bits(p) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_get32bits(p) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) - { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -unsigned int XXH32 (const void* input, size_t len, unsigned seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) - { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do - { - v1 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - v2 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - v3 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - v4 += XXH_get64bits(p) * PRIME64_2; - p+=8; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - } - while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64 * PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64 * PRIME64_1 + PRIME64_4; - } - else - { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) - { - U64 k1 = XXH_get64bits(p); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) - { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/**************************************************** - * Advanced Hash Functions -****************************************************/ - -/*** Allocation ***/ -typedef struct -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - U32 mem32[4]; /* defined as U32 for alignment */ - U32 memsize; -} XXH_istate32_t; - -typedef struct -{ - U64 total_len; - U64 seed; - U64 v1; - U64 v2; - U64 v3; - U64 v4; - U64 mem64[4]; /* defined as U64 for alignment */ - U32 memsize; -} XXH_istate64_t; - - -XXH32_state_t* XXH32_createState(void) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -}; - -XXH64_state_t* XXH64_createState(void) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -}; - - -/*** Hash feed ***/ - -XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) -{ - XXH_istate32_t* state = (XXH_istate32_t*) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - -XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) -{ - XXH_istate64_t* state = (XXH_istate64_t*) state_in; - state->seed = seed; - state->v1 = seed + PRIME64_1 + PRIME64_2; - state->v2 = seed + PRIME64_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME64_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) -{ - XXH_istate32_t* state = (XXH_istate32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { - const U32* p32 = state->mem32; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v1 = XXH_rotl32(state->v1, 13); - state->v1 *= PRIME32_1; - p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v2 = XXH_rotl32(state->v2, 13); - state->v2 *= PRIME32_1; - p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v3 = XXH_rotl32(state->v3, 13); - state->v3 *= PRIME32_1; - p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; - state->v4 = XXH_rotl32(state->v4, 13); - state->v4 *= PRIME32_1; - p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32(p, endian) * PRIME32_2; - v1 = XXH_rotl32(v1, 13); - v1 *= PRIME32_1; - p+=4; - v2 += XXH_readLE32(p, endian) * PRIME32_2; - v2 = XXH_rotl32(v2, 13); - v2 *= PRIME32_1; - p+=4; - v3 += XXH_readLE32(p, endian) * PRIME32_2; - v3 = XXH_rotl32(v3, 13); - v3 *= PRIME32_1; - p+=4; - v4 += XXH_readLE32(p, endian) * PRIME32_2; - v4 = XXH_rotl32(v4, 13); - v4 *= PRIME32_1; - p+=4; - } - while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->mem32, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) -{ - XXH_istate32_t* state = (XXH_istate32_t*) state_in; - const BYTE * p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p+4<=bEnd) - { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) -{ - XXH_istate64_t * state = (XXH_istate64_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) // fill in tmp buffer - { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - { - const U64* p64 = state->mem64; - state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v1 = XXH_rotl64(state->v1, 31); - state->v1 *= PRIME64_1; - p64++; - state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v2 = XXH_rotl64(state->v2, 31); - state->v2 *= PRIME64_1; - p64++; - state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v3 = XXH_rotl64(state->v3, 31); - state->v3 *= PRIME64_1; - p64++; - state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; - state->v4 = XXH_rotl64(state->v4, 31); - state->v4 *= PRIME64_1; - p64++; - } - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) - { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do - { - v1 += XXH_readLE64(p, endian) * PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - p+=8; - v2 += XXH_readLE64(p, endian) * PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - p+=8; - v3 += XXH_readLE64(p, endian) * PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - p+=8; - v4 += XXH_readLE64(p, endian) * PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - p+=8; - } - while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->mem64, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) -{ - XXH_istate64_t * state = (XXH_istate64_t *) state_in; - const BYTE * p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) - { - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - - v1 *= PRIME64_2; - v1 = XXH_rotl64(v1, 31); - v1 *= PRIME64_1; - h64 ^= v1; - h64 = h64*PRIME64_1 + PRIME64_4; - - v2 *= PRIME64_2; - v2 = XXH_rotl64(v2, 31); - v2 *= PRIME64_1; - h64 ^= v2; - h64 = h64*PRIME64_1 + PRIME64_4; - - v3 *= PRIME64_2; - v3 = XXH_rotl64(v3, 31); - v3 *= PRIME64_1; - h64 ^= v3; - h64 = h64*PRIME64_1 + PRIME64_4; - - v4 *= PRIME64_2; - v4 = XXH_rotl64(v4, 31); - v4 *= PRIME64_1; - h64 ^= v4; - h64 = h64*PRIME64_1 + PRIME64_4; - } - else - { - h64 = state->seed + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) - { - U64 k1 = XXH_readLE64(p, endian); - k1 *= PRIME64_2; - k1 = XXH_rotl64(k1,31); - k1 *= PRIME64_1; - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) - { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - diff --git a/xxhash.h b/xxhash.h deleted file mode 100644 index 55b4501..0000000 --- a/xxhash.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -/***************************** - Includes -*****************************/ -#include /* size_t */ - - -/***************************** - Type -*****************************/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -/***************************** - Simple Hash Functions -*****************************/ - -unsigned int XXH32 (const void* input, size_t length, unsigned seed); -unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". -*/ - - - -/***************************** - Advanced Hash Functions -*****************************/ -typedef struct { long long ll[ 6]; } XXH32_state_t; -typedef struct { long long ll[11]; } XXH64_state_t; - -/* -These structures allow static allocation of XXH states. -States must then be initialized using XXHnn_reset() before first use. - -If you prefer dynamic allocation, please refer to functions below. -*/ - -XXH32_state_t* XXH32_createState(void); -XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH64_state_t* XXH64_createState(void); -XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - -/* -These functions create and release memory for XXH state. -States must then be initialized using XXHnn_reset() before first use. -*/ - - -XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); -XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -unsigned int XXH32_digest (const XXH32_state_t* statePtr); - -XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -unsigned long long XXH64_digest (const XXH64_state_t* statePtr); - -/* -These functions calculate the xxHash of an input provided in multiple smaller packets, -as opposed to an input provided as a single block. - -XXH state space must first be allocated, using either static or dynamic method provided above. - -Start a new hash by initializing state with a seed, using XXHnn_reset(). - -Then, feed the hash state by calling XXHnn_update() as many times as necessary. -Obviously, input must be valid, meaning allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, you can produce a hash anytime, by using XXHnn_digest(). -This function returns the final nn-bits hash. -You can nonetheless continue feeding the hash state with more input, -and therefore get some new hashes, by calling again XXHnn_digest(). - -When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). -*/ - - -#if defined (__cplusplus) -} -#endif -- cgit v0.12 From bf79270cedeb1faa50f2f0ef62039c200ec5cd39 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Nov 2014 23:34:29 +0100 Subject: --- lib/liblz4.pc | 14 -------------- lib/liblz4.so | 1 - lib/liblz4.so.1 | 1 - lib/liblz4.so.1.4.1 | Bin 68571 -> 0 bytes lib/lz4.o | Bin 47472 -> 0 bytes lib/lz4hc.o | Bin 15888 -> 0 bytes 6 files changed, 16 deletions(-) delete mode 100644 lib/liblz4.pc delete mode 120000 lib/liblz4.so delete mode 120000 lib/liblz4.so.1 delete mode 100755 lib/liblz4.so.1.4.1 delete mode 100644 lib/lz4.o delete mode 100644 lib/lz4hc.o diff --git a/lib/liblz4.pc b/lib/liblz4.pc deleted file mode 100644 index 39983ba..0000000 --- a/lib/liblz4.pc +++ /dev/null @@ -1,14 +0,0 @@ -# LZ4 - Fast LZ compression algorithm -# Copyright (C) 2011-2014, Yann Collet. -# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -prefix=/usr -libdir=/usr/lib -includedir=/usr/include - -Name: lz4 -Description: fast lossless compression algorithm library -URL: http://code.google.com/p/lz4/ -Version: -Libs: -L/usr/lib -llz4 -Cflags: -I/usr/include diff --git a/lib/liblz4.so b/lib/liblz4.so deleted file mode 120000 index 3c57595..0000000 --- a/lib/liblz4.so +++ /dev/null @@ -1 +0,0 @@ -liblz4.so.1.4.1 \ No newline at end of file diff --git a/lib/liblz4.so.1 b/lib/liblz4.so.1 deleted file mode 120000 index 3c57595..0000000 --- a/lib/liblz4.so.1 +++ /dev/null @@ -1 +0,0 @@ -liblz4.so.1.4.1 \ No newline at end of file diff --git a/lib/liblz4.so.1.4.1 b/lib/liblz4.so.1.4.1 deleted file mode 100755 index 6ff6fcf..0000000 Binary files a/lib/liblz4.so.1.4.1 and /dev/null differ diff --git a/lib/lz4.o b/lib/lz4.o deleted file mode 100644 index 2ed244d..0000000 Binary files a/lib/lz4.o and /dev/null differ diff --git a/lib/lz4hc.o b/lib/lz4hc.o deleted file mode 100644 index bbc0911..0000000 Binary files a/lib/lz4hc.o and /dev/null differ -- cgit v0.12 From 989346b75eb82f1d3504bec82e4ede77c11d5711 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 30 Nov 2014 23:37:55 +0100 Subject: fix : sudo for make install --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 60f8ce1..d8c2621 100644 --- a/Makefile +++ b/Makefile @@ -93,8 +93,8 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) install: - @cd $(LZ4DIR); $(MAKE) -e install - @cd $(PRGDIR); $(MAKE) -e install + @cd $(LZ4DIR); sudo $(MAKE) -e install + @cd $(PRGDIR); sudo $(MAKE) -e install uninstall: @cd $(LZ4DIR); $(MAKE) uninstall -- cgit v0.12 From 9523101aff2f7d38572b22b522430b9af39df873 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 1 Dec 2014 00:53:20 +0100 Subject: Clarified some file names --- LZ4_Frame_Format.html | 1 + LZ4_Framing_Format.html | 1 - liblz4.pc.in | 14 ------ lz4_block_format.txt | 120 +++++++++++++++++++++++++++++++++++++++++++++ lz4_format_description.txt | 120 --------------------------------------------- 5 files changed, 121 insertions(+), 135 deletions(-) create mode 100644 LZ4_Frame_Format.html delete mode 100644 LZ4_Framing_Format.html delete mode 100644 liblz4.pc.in create mode 100644 lz4_block_format.txt delete mode 100644 lz4_format_description.txt diff --git a/LZ4_Frame_Format.html b/LZ4_Frame_Format.html new file mode 100644 index 0000000..bd1304c --- /dev/null +++ b/LZ4_Frame_Format.html @@ -0,0 +1 @@ +LZ4 Framing format - stable

LZ4 Framing Format

Notices

Copyright (c) 2013-2014 Yann Collet

Permission is granted to copy and distribute this document for any  purpose and without charge, including translations into other  languages and incorporation into compilations, provided that the copyright notice and this notice are preserved, and that any substantive changes or deletions from the original are clearly marked.

Version

1.4.1

Introduction

The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for File compression, Pipe and streaming compression using the LZ4 algorithm : http://code.google.com/p/lz4/

The data can be produced or consumed, even for an arbitrarily long sequentially presented input data stream, using only an a priori bounded amount of intermediate storage, and hence can be used in data communications.  The format uses the LZ4 compression method, and xxHash-32 checksum method, for detection of data corruption.

The data format defined by this specification does not attempt to allow random access to compressed data.

This specification is intended for use by implementers of software to compress data into LZ4 format and/or decompress data from LZ4 format. The text of the specification assumes a basic background in programming at the level of bits and other primitive data representations.

Unless otherwise indicated below, a compliant compressor must produce data sets that conform to all the specifications presented here.

A compliant decompressor must be able to accept and decompress at least one data set that conforms to the specifications presented here; whenever it does not support any parameter, it must produce a non-ambiguous error code and associated error message explaining which parameter value is unsupported (a typical example being an unsupported buffer size).

Distribution of this document is unlimited.

Summary :

Introduction

General structure of LZ4 Framing Format

Frame Descriptor

Data Blocks

Skippable Frames

Legacy format

Appendix

General Structure of LZ4 Framing format

LZ4 Framing Format - General Structure.png

Magic Number

4 Bytes, Little endian format.
Value : 0x184D2204

Frame Descriptor

3 to 15 Bytes, to be detailed in the next part.
Most significant part of the spec.

Data Blocks

To be detailed later on.
That’s where compressed data is stored.

EndMark

The flow of blocks ends when the last data block has a size of “0”.
The size is expressed as a 32-bits value.

Content Checksum

Content Checksum verify that the full content has been decoded correctly.
The content checksum is the result of xxh32() hash function digesting the original (decoded) data as input, and a seed of zero.
Content checksum is only present when its associated flag is set in the framing descriptor. Content Checksum validates the result, that all blocks were fully transmitted in the correct order and without error, and also that the encoding/decoding process itself generated no distortion. Its usage is recommended.

Frame Concatenation

In some circumstances, it may be preferable to append multiple frames, for example in order to add new data to an existing compressed file without re-framing it.

In such case, each frame has its own set of descriptor flags. Each frame is considered independent. The only relation between frames is their sequential order.

The ability to decode multiple concatenated frames within a single stream or file is left outside of this specification. While a logical default behavior could be to decode the frames in their sequential order, this is not a requirement.

Frame Descriptor

LZ4 Framing Format - Frame Descriptor.png

LZ4 Framing Format - Descriptor Flags.png

The descriptor uses a minimum of 3 bytes, and up to 15 bytes depending on optional parameters.
In the picture, bit 7 is highest bit, while bit 0 is lowest.

Version Number :

2-bits field, must be set to “01”.
Any other value cannot be decoded by this version of the specification.
Other version numbers will use different flag layouts.

Block Independence flag :

If this flag is set to “1”, blocks are independent, and can therefore be decoded independently, in parallel.
If this flag is set to “0”, each block depends on previous ones for decoding (up to LZ4 window size, which is 64 KB). In this case, it’s necessary to decode all blocks in sequence.

Block dependency improves compression ratio, especially for small blocks. On the other hand, it makes jumps or multi-threaded decoding impossible.

Block checksum flag :

If this flag is set, each data block will be followed by a 4-bytes checksum, calculated by using the xxHash-32 algorithm on the raw (compressed) data block.
The intention is to detect data corruption (storage or transmission errors) immediately, before decoding.
Block checksum usage is optional.

Content Size flag :

If this flag is set, the original (uncompressed) size of data included within the frame will be present as an 8 bytes unsigned value, little endian format, after the flags.

Recommended value : “0” (not present)

Content checksum flag :

If this flag is set, a content checksum will be appended after the EoS mark.

Recommended value : “1” (content checksum is present)

Preset Dictionary flag :

If this flag is set, a Dict-ID field will be present, just after the descriptor flags and the Content size.

Usual value : “0” (not present)

Block Maximum Size :

This information is intended to help the decoder allocate the right amount of memory.
Size here refers to the original (uncompressed) data size.
Block Maximum Size is one value among the following table :

The decoder may refuse to allocate block sizes above a (system-specific) size.
Unused values may be used in a future revision of the spec.
A decoder conformant to the current version of the spec is only able to decode blocksizes defined in this spec.

Reserved bits :

Value of reserved bits must be 0 (zero).
Reserved bit might be used in a future version of the specification, to enable any (yet-to-decide) optional feature.
If this happens, a decoder respecting the current version of the specification shall not be able to decode such a frame.

Content Size

This is the original (uncompressed) size.
This information is optional, and only present if the associated flag is set.
Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.
Format is Little endian.
This field has no impact on decoding, it just informs the decoder how much data the frame holds (for example, to display it during decoding process, or for verification purpose). It can be safely skipped by a conformant decoder.

Dictionary ID

Dict-ID is only present if the associated flag is set.
A dictionary is specially useful to compress short input sequences. The compressor can take advantage of the dictionary context to encode the input in a more compact manner. It works as a kind of “known prefix” which is used by both the compressor and the decompressor to “warm-up” reference tables and help compress small data blocks.

Dict-ID is the xxHash-32 checksum of this “known prefix”. Format is Little endian.

The decompressor uses this identifier to determine which dictionary has been used by the compressor. The compressor and the decompressor must use exactly the same dictionary. This document does not specify the contents of predefined dictionaries, since the optimal dictionaries are application specific. Any data format using this feature must precisely define the allowed dictionaries.

Within a single frame, a single dictionary can be defined.
When the frame descriptor defines independent blocks, each block will be initialised with the same dictionary.
If the frame descriptor defines linked blocks, the dictionary will only be used once, at the beginning of the decoding process.

Header Checksum :

One-byte checksum of all descriptor fields, including optional ones when present.
The byte is second byte of xxh32() : { (xxh32()>>8) & 0xFF } ,
using zero as a seed,
and the full Frame Descriptor as an input (including optional fields when they are present).
A different checksum indicates an error in the descriptor.

Data Blocks

Block Size

This field uses 4-bytes,  format is little-endian.

The highest bit is “1” if data in the block is uncompressed.

The highest bit is “0” if data in the block is compressed by LZ4.

All other bits give the size, in bytes, of the following data block (the size does not include the checksum if present).

Block Size shall never be larger than Block Maximum Size. Such a thing could happen when the original data is incompressible. In this case, such a data block shall be passed in uncompressed format.

Data

Where the actual data to decode stands. It might be compressed or not, depending on previous field indications.
Uncompressed size of Data can be any size, up to “block maximum size”.
Note that the data block is not necessarily filled : an arbitrary “flush” may happen anytime. Any block can be “partially filled”.

Block checksum :

Only present if the associated flag is set.
This is a 4-bytes checksum value, in little endian format,
calculated by using the xxHash-32 algorithm on the raw (undecoded) data block,
and a seed of zero.
The intention is to detect data corruption (storage or transmission errors) before decoding.

Block checksum is cumulative with Content checksum.

Skippable Frames

LZ4 Framing Format - Skippable Frame.png

Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
Its design is pretty straightforward, with the sole objective to allow the decoder to quickly skip over user-defined data and continue decoding.

For the purpose of facilitating identification, it is discouraged to start a flow of concatenated frames with a skippable frame. If there is a need to start such a flow with some user data encapsulated into a skippable frame, it’s recommended to start will a zero-byte LZ4 frame followed by a skippable frame. This will make it easier for file type identifiers.

 

Magic Number

4 Bytes, Little endian format.
Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a skippable frame.

Frame Size 

This is the size, in bytes, of the following User Data (without including the magic number nor the size field itself).
4 Bytes, Little endian format, unsigned 32-bits.
This means User Data can’t be bigger than (2^32-1) Bytes.

User Data

User Data can be anything. Data will just be skipped by the decoder.

Legacy frame

The Legacy frame format was defined into the initial versions of “LZ4Demo”.
Newer compressors should not use this format anymore, since it is too restrictive.
It is recommended that decompressors shall be able to decode this format during the transition period.

Main properties of legacy format :
- Fixed block size : 8 MB.
- All blocks must be completely filled, except the last one.
- All blocks are always compressed, even when compression is detrimental.
- The last block is detected either because it is followed by the “EOF” (End of File) mark, or because it is followed by a known Frame Magic Number.
- No checksum
- Convention is Little endian

Magic Number

4 Bytes, Little endian format.
Value : 0x184C2102

Block Compressed Size

This is the size, in bytes, of the following compressed data block.
4 Bytes, Little endian format.

Data

Where the actual data stands.
Data is always compressed, even when compression is detrimental (i.e. larger than original size).

Appendix  

Version changes

1.4.1 : changed wording from “stream” to “frame”

1.4 : added skippable streams, re-added stream checksum

1.3 : modified header checksum

1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”.

1.1 : optional fields are now part of the descriptor

1.0 : changed “block size” specification, adding a compressed/uncompressed flag

0.9 : reduced scale of “block maximum size” table

0.8 : removed : high compression flag

0.7 : removed : stream checksum

0.6 : settled : stream size uses 8 bytes, endian convention is little endian

0.5: added copyright notice

0.4 : changed format to Google Doc compatible OpenDocument

\ No newline at end of file diff --git a/LZ4_Framing_Format.html b/LZ4_Framing_Format.html deleted file mode 100644 index bd1304c..0000000 --- a/LZ4_Framing_Format.html +++ /dev/null @@ -1 +0,0 @@ -LZ4 Framing format - stable

LZ4 Framing Format

Notices

Copyright (c) 2013-2014 Yann Collet

Permission is granted to copy and distribute this document for any  purpose and without charge, including translations into other  languages and incorporation into compilations, provided that the copyright notice and this notice are preserved, and that any substantive changes or deletions from the original are clearly marked.

Version

1.4.1

Introduction

The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for File compression, Pipe and streaming compression using the LZ4 algorithm : http://code.google.com/p/lz4/

The data can be produced or consumed, even for an arbitrarily long sequentially presented input data stream, using only an a priori bounded amount of intermediate storage, and hence can be used in data communications.  The format uses the LZ4 compression method, and xxHash-32 checksum method, for detection of data corruption.

The data format defined by this specification does not attempt to allow random access to compressed data.

This specification is intended for use by implementers of software to compress data into LZ4 format and/or decompress data from LZ4 format. The text of the specification assumes a basic background in programming at the level of bits and other primitive data representations.

Unless otherwise indicated below, a compliant compressor must produce data sets that conform to all the specifications presented here.

A compliant decompressor must be able to accept and decompress at least one data set that conforms to the specifications presented here; whenever it does not support any parameter, it must produce a non-ambiguous error code and associated error message explaining which parameter value is unsupported (a typical example being an unsupported buffer size).

Distribution of this document is unlimited.

Summary :

Introduction

General structure of LZ4 Framing Format

Frame Descriptor

Data Blocks

Skippable Frames

Legacy format

Appendix

General Structure of LZ4 Framing format

LZ4 Framing Format - General Structure.png

Magic Number

4 Bytes, Little endian format.
Value : 0x184D2204

Frame Descriptor

3 to 15 Bytes, to be detailed in the next part.
Most significant part of the spec.

Data Blocks

To be detailed later on.
That’s where compressed data is stored.

EndMark

The flow of blocks ends when the last data block has a size of “0”.
The size is expressed as a 32-bits value.

Content Checksum

Content Checksum verify that the full content has been decoded correctly.
The content checksum is the result of xxh32() hash function digesting the original (decoded) data as input, and a seed of zero.
Content checksum is only present when its associated flag is set in the framing descriptor. Content Checksum validates the result, that all blocks were fully transmitted in the correct order and without error, and also that the encoding/decoding process itself generated no distortion. Its usage is recommended.

Frame Concatenation

In some circumstances, it may be preferable to append multiple frames, for example in order to add new data to an existing compressed file without re-framing it.

In such case, each frame has its own set of descriptor flags. Each frame is considered independent. The only relation between frames is their sequential order.

The ability to decode multiple concatenated frames within a single stream or file is left outside of this specification. While a logical default behavior could be to decode the frames in their sequential order, this is not a requirement.

Frame Descriptor

LZ4 Framing Format - Frame Descriptor.png

LZ4 Framing Format - Descriptor Flags.png

The descriptor uses a minimum of 3 bytes, and up to 15 bytes depending on optional parameters.
In the picture, bit 7 is highest bit, while bit 0 is lowest.

Version Number :

2-bits field, must be set to “01”.
Any other value cannot be decoded by this version of the specification.
Other version numbers will use different flag layouts.

Block Independence flag :

If this flag is set to “1”, blocks are independent, and can therefore be decoded independently, in parallel.
If this flag is set to “0”, each block depends on previous ones for decoding (up to LZ4 window size, which is 64 KB). In this case, it’s necessary to decode all blocks in sequence.

Block dependency improves compression ratio, especially for small blocks. On the other hand, it makes jumps or multi-threaded decoding impossible.

Block checksum flag :

If this flag is set, each data block will be followed by a 4-bytes checksum, calculated by using the xxHash-32 algorithm on the raw (compressed) data block.
The intention is to detect data corruption (storage or transmission errors) immediately, before decoding.
Block checksum usage is optional.

Content Size flag :

If this flag is set, the original (uncompressed) size of data included within the frame will be present as an 8 bytes unsigned value, little endian format, after the flags.

Recommended value : “0” (not present)

Content checksum flag :

If this flag is set, a content checksum will be appended after the EoS mark.

Recommended value : “1” (content checksum is present)

Preset Dictionary flag :

If this flag is set, a Dict-ID field will be present, just after the descriptor flags and the Content size.

Usual value : “0” (not present)

Block Maximum Size :

This information is intended to help the decoder allocate the right amount of memory.
Size here refers to the original (uncompressed) data size.
Block Maximum Size is one value among the following table :

The decoder may refuse to allocate block sizes above a (system-specific) size.
Unused values may be used in a future revision of the spec.
A decoder conformant to the current version of the spec is only able to decode blocksizes defined in this spec.

Reserved bits :

Value of reserved bits must be 0 (zero).
Reserved bit might be used in a future version of the specification, to enable any (yet-to-decide) optional feature.
If this happens, a decoder respecting the current version of the specification shall not be able to decode such a frame.

Content Size

This is the original (uncompressed) size.
This information is optional, and only present if the associated flag is set.
Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.
Format is Little endian.
This field has no impact on decoding, it just informs the decoder how much data the frame holds (for example, to display it during decoding process, or for verification purpose). It can be safely skipped by a conformant decoder.

Dictionary ID

Dict-ID is only present if the associated flag is set.
A dictionary is specially useful to compress short input sequences. The compressor can take advantage of the dictionary context to encode the input in a more compact manner. It works as a kind of “known prefix” which is used by both the compressor and the decompressor to “warm-up” reference tables and help compress small data blocks.

Dict-ID is the xxHash-32 checksum of this “known prefix”. Format is Little endian.

The decompressor uses this identifier to determine which dictionary has been used by the compressor. The compressor and the decompressor must use exactly the same dictionary. This document does not specify the contents of predefined dictionaries, since the optimal dictionaries are application specific. Any data format using this feature must precisely define the allowed dictionaries.

Within a single frame, a single dictionary can be defined.
When the frame descriptor defines independent blocks, each block will be initialised with the same dictionary.
If the frame descriptor defines linked blocks, the dictionary will only be used once, at the beginning of the decoding process.

Header Checksum :

One-byte checksum of all descriptor fields, including optional ones when present.
The byte is second byte of xxh32() : { (xxh32()>>8) & 0xFF } ,
using zero as a seed,
and the full Frame Descriptor as an input (including optional fields when they are present).
A different checksum indicates an error in the descriptor.

Data Blocks

Block Size

This field uses 4-bytes,  format is little-endian.

The highest bit is “1” if data in the block is uncompressed.

The highest bit is “0” if data in the block is compressed by LZ4.

All other bits give the size, in bytes, of the following data block (the size does not include the checksum if present).

Block Size shall never be larger than Block Maximum Size. Such a thing could happen when the original data is incompressible. In this case, such a data block shall be passed in uncompressed format.

Data

Where the actual data to decode stands. It might be compressed or not, depending on previous field indications.
Uncompressed size of Data can be any size, up to “block maximum size”.
Note that the data block is not necessarily filled : an arbitrary “flush” may happen anytime. Any block can be “partially filled”.

Block checksum :

Only present if the associated flag is set.
This is a 4-bytes checksum value, in little endian format,
calculated by using the xxHash-32 algorithm on the raw (undecoded) data block,
and a seed of zero.
The intention is to detect data corruption (storage or transmission errors) before decoding.

Block checksum is cumulative with Content checksum.

Skippable Frames

LZ4 Framing Format - Skippable Frame.png

Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
Its design is pretty straightforward, with the sole objective to allow the decoder to quickly skip over user-defined data and continue decoding.

For the purpose of facilitating identification, it is discouraged to start a flow of concatenated frames with a skippable frame. If there is a need to start such a flow with some user data encapsulated into a skippable frame, it’s recommended to start will a zero-byte LZ4 frame followed by a skippable frame. This will make it easier for file type identifiers.

 

Magic Number

4 Bytes, Little endian format.
Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a skippable frame.

Frame Size 

This is the size, in bytes, of the following User Data (without including the magic number nor the size field itself).
4 Bytes, Little endian format, unsigned 32-bits.
This means User Data can’t be bigger than (2^32-1) Bytes.

User Data

User Data can be anything. Data will just be skipped by the decoder.

Legacy frame

The Legacy frame format was defined into the initial versions of “LZ4Demo”.
Newer compressors should not use this format anymore, since it is too restrictive.
It is recommended that decompressors shall be able to decode this format during the transition period.

Main properties of legacy format :
- Fixed block size : 8 MB.
- All blocks must be completely filled, except the last one.
- All blocks are always compressed, even when compression is detrimental.
- The last block is detected either because it is followed by the “EOF” (End of File) mark, or because it is followed by a known Frame Magic Number.
- No checksum
- Convention is Little endian

Magic Number

4 Bytes, Little endian format.
Value : 0x184C2102

Block Compressed Size

This is the size, in bytes, of the following compressed data block.
4 Bytes, Little endian format.

Data

Where the actual data stands.
Data is always compressed, even when compression is detrimental (i.e. larger than original size).

Appendix  

Version changes

1.4.1 : changed wording from “stream” to “frame”

1.4 : added skippable streams, re-added stream checksum

1.3 : modified header checksum

1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”.

1.1 : optional fields are now part of the descriptor

1.0 : changed “block size” specification, adding a compressed/uncompressed flag

0.9 : reduced scale of “block maximum size” table

0.8 : removed : high compression flag

0.7 : removed : stream checksum

0.6 : settled : stream size uses 8 bytes, endian convention is little endian

0.5: added copyright notice

0.4 : changed format to Google Doc compatible OpenDocument

\ No newline at end of file diff --git a/liblz4.pc.in b/liblz4.pc.in deleted file mode 100644 index 0d05152..0000000 --- a/liblz4.pc.in +++ /dev/null @@ -1,14 +0,0 @@ -# LZ4 - Fast LZ compression algorithm -# Copyright (C) 2011-2014, Yann Collet. -# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ - -Name: lz4 -Description: fast lossless compression algorithm library -URL: http://code.google.com/p/lz4/ -Version: @VERSION@ -Libs: -L@LIBDIR@ -llz4 -Cflags: -I@INCLUDEDIR@ diff --git a/lz4_block_format.txt b/lz4_block_format.txt new file mode 100644 index 0000000..2c424c5 --- /dev/null +++ b/lz4_block_format.txt @@ -0,0 +1,120 @@ +LZ4 Format Description +Last revised: 2012-02-27 +Author : Y. Collet + + + +This small specification intents to provide enough information +to anyone willing to produce LZ4-compatible compressed data blocks +using any programming language. + +LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. +The most important design principle behind LZ4 is simplicity. +It helps to create an easy to read and maintain source code. +It also helps later on for optimisations, compactness, and speed. +There is no entropy encoder backend nor framing layer. +The latter is assumed to be handled by other parts of the system. + +This document only describes the format, +not how the LZ4 compressor nor decompressor actually work. +The correctness of the decompressor should not depend +on implementation details of the compressor, and vice versa. + + + +-- Compressed block format -- + +An LZ4 compressed block is composed of sequences. +Schematically, a sequence is a suite of literals, followed by a match copy. + +Each sequence starts with a token. +The token is a one byte value, separated into two 4-bits fields. +Therefore each field ranges from 0 to 15. + + +The first field uses the 4 high-bits of the token. +It provides the length of literals to follow. +(Note : a literal is a not-compressed byte). +If the field value is 0, then there is no literal. +If it is 15, then we need to add some more bytes to indicate the full length. +Each additionnal byte then represent a value from 0 to 255, +which is added to the previous value to produce a total length. +When the byte value is 255, another byte is output. +There can be any number of bytes following the token. There is no "size limit". +(Sidenote this is why a not-compressible input block is expanded by 0.4%). + +Example 1 : A length of 48 will be represented as : +- 15 : value for the 4-bits High field +- 33 : (=48-15) remaining length to reach 48 + +Example 2 : A length of 280 will be represented as : +- 15 : value for the 4-bits High field +- 255 : following byte is maxed, since 280-15 >= 255 +- 10 : (=280 - 15 - 255) ) remaining length to reach 280 + +Example 3 : A length of 15 will be represented as : +- 15 : value for the 4-bits High field +- 0 : (=15-15) yes, the zero must be output + +Following the token and optional length bytes, are the literals themselves. +They are exactly as numerous as previously decoded (length of literals). +It's possible that there are zero literal. + + +Following the literals is the match copy operation. + +It starts by the offset. +This is a 2 bytes value, in little endian format. + +The offset represents the position of the match to be copied from. +1 means "current position - 1 byte". +The maximum offset value is 65535, 65536 cannot be coded. +Note that 0 is an invalid value, not used. + +Then we need to extract the match length. +For this, we use the second token field, the low 4-bits. +Value, obviously, ranges from 0 to 15. +However here, 0 means that the copy operation will be minimal. +The minimum length of a match, called minmatch, is 4. +As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. +Similar to literal length, on reaching the highest possible value (15), +we output additional bytes, one at a time, with values ranging from 0 to 255. +They are added to total to provide the final match length. +A 255 value means there is another byte to read and add. +There is no limit to the number of optional bytes that can be output this way. +(This points towards a maximum achievable compression ratio of ~250). + +With the offset and the matchlength, +the decoder can now proceed to copy the data from the already decoded buffer. +On decoding the matchlength, we reach the end of the compressed sequence, +and therefore start another one. + + +-- Parsing restrictions -- + +There are specific parsing rules to respect in order to remain compatible +with assumptions made by the decoder : +1) The last 5 bytes are always literals +2) The last match must start at least 12 bytes before end of block +Consequently, a block with less than 13 bytes cannot be compressed. +These rules are in place to ensure that the decoder +will never read beyond the input buffer, nor write beyond the output buffer. + +Note that the last sequence is also incomplete, +and stops right after literals. + + +-- Additional notes -- + +There is no assumption nor limits to the way the compressor +searches and selects matches within the source data block. +It could be a fast scan, a multi-probe, a full search using BST, +standard hash chains or MMC, well whatever. + +Advanced parsing strategies can also be implemented, such as lazy match, +or full optimal parsing. + +All these trade-off offer distinctive speed/memory/compression advantages. +Whatever the method used by the compressor, its result will be decodable +by any LZ4 decoder if it follows the format specification described above. + diff --git a/lz4_format_description.txt b/lz4_format_description.txt deleted file mode 100644 index 2c424c5..0000000 --- a/lz4_format_description.txt +++ /dev/null @@ -1,120 +0,0 @@ -LZ4 Format Description -Last revised: 2012-02-27 -Author : Y. Collet - - - -This small specification intents to provide enough information -to anyone willing to produce LZ4-compatible compressed data blocks -using any programming language. - -LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. -The most important design principle behind LZ4 is simplicity. -It helps to create an easy to read and maintain source code. -It also helps later on for optimisations, compactness, and speed. -There is no entropy encoder backend nor framing layer. -The latter is assumed to be handled by other parts of the system. - -This document only describes the format, -not how the LZ4 compressor nor decompressor actually work. -The correctness of the decompressor should not depend -on implementation details of the compressor, and vice versa. - - - --- Compressed block format -- - -An LZ4 compressed block is composed of sequences. -Schematically, a sequence is a suite of literals, followed by a match copy. - -Each sequence starts with a token. -The token is a one byte value, separated into two 4-bits fields. -Therefore each field ranges from 0 to 15. - - -The first field uses the 4 high-bits of the token. -It provides the length of literals to follow. -(Note : a literal is a not-compressed byte). -If the field value is 0, then there is no literal. -If it is 15, then we need to add some more bytes to indicate the full length. -Each additionnal byte then represent a value from 0 to 255, -which is added to the previous value to produce a total length. -When the byte value is 255, another byte is output. -There can be any number of bytes following the token. There is no "size limit". -(Sidenote this is why a not-compressible input block is expanded by 0.4%). - -Example 1 : A length of 48 will be represented as : -- 15 : value for the 4-bits High field -- 33 : (=48-15) remaining length to reach 48 - -Example 2 : A length of 280 will be represented as : -- 15 : value for the 4-bits High field -- 255 : following byte is maxed, since 280-15 >= 255 -- 10 : (=280 - 15 - 255) ) remaining length to reach 280 - -Example 3 : A length of 15 will be represented as : -- 15 : value for the 4-bits High field -- 0 : (=15-15) yes, the zero must be output - -Following the token and optional length bytes, are the literals themselves. -They are exactly as numerous as previously decoded (length of literals). -It's possible that there are zero literal. - - -Following the literals is the match copy operation. - -It starts by the offset. -This is a 2 bytes value, in little endian format. - -The offset represents the position of the match to be copied from. -1 means "current position - 1 byte". -The maximum offset value is 65535, 65536 cannot be coded. -Note that 0 is an invalid value, not used. - -Then we need to extract the match length. -For this, we use the second token field, the low 4-bits. -Value, obviously, ranges from 0 to 15. -However here, 0 means that the copy operation will be minimal. -The minimum length of a match, called minmatch, is 4. -As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. -Similar to literal length, on reaching the highest possible value (15), -we output additional bytes, one at a time, with values ranging from 0 to 255. -They are added to total to provide the final match length. -A 255 value means there is another byte to read and add. -There is no limit to the number of optional bytes that can be output this way. -(This points towards a maximum achievable compression ratio of ~250). - -With the offset and the matchlength, -the decoder can now proceed to copy the data from the already decoded buffer. -On decoding the matchlength, we reach the end of the compressed sequence, -and therefore start another one. - - --- Parsing restrictions -- - -There are specific parsing rules to respect in order to remain compatible -with assumptions made by the decoder : -1) The last 5 bytes are always literals -2) The last match must start at least 12 bytes before end of block -Consequently, a block with less than 13 bytes cannot be compressed. -These rules are in place to ensure that the decoder -will never read beyond the input buffer, nor write beyond the output buffer. - -Note that the last sequence is also incomplete, -and stops right after literals. - - --- Additional notes -- - -There is no assumption nor limits to the way the compressor -searches and selects matches within the source data block. -It could be a fast scan, a multi-probe, a full search using BST, -standard hash chains or MMC, well whatever. - -Advanced parsing strategies can also be implemented, such as lazy match, -or full optimal parsing. - -All these trade-off offer distinctive speed/memory/compression advantages. -Whatever the method used by the compressor, its result will be decodable -by any LZ4 decoder if it follows the format specification described above. - -- cgit v0.12 From a109c91818341188d8d64096d1701c1747ed480d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 1 Dec 2014 01:20:42 +0100 Subject: Fixed : make dist --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d8c2621..66c8c4b 100644 --- a/Makefile +++ b/Makefile @@ -49,15 +49,14 @@ DISTRIBNAME=lz4-$(RELEASE).tar.gz TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4frame.h $(LZ4DIR)/xxhash.c $(LZ4DIR)/xxhash.h \ $(LZ4DIR)/liblz4.pc.in $(LZ4DIR)/Makefile $(LZ4DIR)/LICENSE \ - Makefile lz4_format_description.txt NEWS README.md \ + Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \ cmake_unofficial/CMakeLists.txt \ $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ $(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \ $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ $(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \ - $(PRGDIR)/Makefile $(PRGDIR)/COPYING \ - LZ4_Framing_Format.html + $(PRGDIR)/Makefile $(PRGDIR)/COPYING NONTEXT = images/image00.png images/image01.png images/image02.png \ images/image03.png images/image04.png images/image05.png \ images/image06.png -- cgit v0.12 From 9f6826fdc9621935324f35a596d43c145f7b172c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 1 Dec 2014 01:25:18 +0100 Subject: lz4cli : legacy arguments are now disabled by default --- programs/Makefile | 6 +++--- programs/lz4cli.c | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 96de0e3..eca920b 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -66,13 +66,13 @@ default: lz4 lz4c all: lz4 lz4c lz4c32 fullbench fullbench32 fuzzer fuzzer32 frametest frametest32 datagen lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c - $(CC) $(FLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) + $(CC) $(FLAGS) $^ -o $@$(EXT) lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(CC) $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c bench.c lz4io.c lz4cli.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + $(CC) -m32 $(FLAGS) -DENABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/xxhash.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index e69a84a..351de80 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -32,10 +32,10 @@ //************************************** // Tuning parameters //************************************** -// DISABLE_LZ4C_LEGACY_OPTIONS : +// ENABLE_LZ4C_LEGACY_OPTIONS : // Control the availability of -c0, -c1 and -hc legacy arguments -// Default : Legacy options are enabled -// #define DISABLE_LZ4C_LEGACY_OPTIONS +// Default : Legacy options are disabled +// #define ENABLE_LZ4C_LEGACY_OPTIONS //************************************** @@ -201,14 +201,14 @@ int usage_advanced(void) DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); -#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) +#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) DISPLAY( "Legacy arguments :\n"); DISPLAY( " -c0 : fast compression\n"); DISPLAY( " -c1 : high compression\n"); DISPLAY( " -hc : high compression\n"); DISPLAY( " -y : overwrite output without prompting \n"); DISPLAY( " -s : suppress warnings \n"); -#endif // DISABLE_LZ4C_LEGACY_OPTIONS +#endif // ENABLE_LZ4C_LEGACY_OPTIONS EXTENDED_HELP; return 0; } @@ -250,7 +250,7 @@ int usage_longhelp(void) DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName); DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); DISPLAY( " generator | %s | consumer \n", programName); -#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) +#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) DISPLAY( "\n"); DISPLAY( "Warning :\n"); DISPLAY( "Legacy arguments take precedence. Therefore : \n"); @@ -259,7 +259,7 @@ int usage_longhelp(void) DISPLAY( "It is not equivalent to :\n"); DISPLAY( " %s -h -c filename\n", programName); DISPLAY( "which would display help text and exit\n"); -#endif // DISABLE_LZ4C_LEGACY_OPTIONS +#endif // ENABLE_LZ4C_LEGACY_OPTIONS return 0; } @@ -325,14 +325,14 @@ int main(int argc, char** argv) { argument ++; -#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) +#if defined(ENABLE_LZ4C_LEGACY_OPTIONS) // Legacy options (-c0, -c1, -hc, -y, -s) if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression) if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression) if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression) if (*argument=='y') { LZ4IO_setOverwrite(1); continue; } // -y (answer 'yes' to overwrite permission) if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode) -#endif // DISABLE_LZ4C_LEGACY_OPTIONS +#endif // ENABLE_LZ4C_LEGACY_OPTIONS if ((*argument>='0') && (*argument<='9')) { -- cgit v0.12 From 67c25a5c1b03eaf61a89c634469fad3b0de76ae8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 1 Dec 2014 22:00:07 +0100 Subject: lz4frame.h : clarified a few comments --- lib/lz4frame.h | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index a4f5e36..7184cc3 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -1,7 +1,7 @@ /* LZ4 auto-framing library Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -29,6 +29,7 @@ You can contact the author at : - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source mirror : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -46,7 +47,7 @@ extern "C" { /**************************************** Note : experimental API. - Not yet integrated within lz4 library. + Not yet integrated within liblz4 ****************************************/ /************************************** @@ -56,8 +57,8 @@ extern "C" { /************************************** - Error management -**************************************/ + * Error management + * ************************************/ typedef size_t LZ4F_errorCode_t; #define LZ4F_LIST_ERRORS(ITEM) \ ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ @@ -72,15 +73,15 @@ typedef size_t LZ4F_errorCode_t; #define LZ4F_GENERATE_ENUM(ENUM) ENUM, typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ -int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ -const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return enum as string */ +int LZ4F_isError(LZ4F_errorCode_t code); /* Basically : code > -ERROR_maxCode */ +const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */ /************************************** - Framing compression functions -**************************************/ + * Frame compression types + * ************************************/ -typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7} blockSizeID_t; +typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t; typedef enum { blockLinked=0, blockIndependent} blockMode_t; typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; @@ -93,8 +94,8 @@ typedef struct { typedef struct { LZ4F_frameInfo_t frameInfo; - unsigned compressionLevel; /* Not yet supported : only fast compression for the time being */ - unsigned autoFlush; /* 1 == always flush; reduce need for tmp buffer */ + unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */ + unsigned autoFlush; /* 1 == always flush : reduce need for tmp buffer */ unsigned reserved[4]; } LZ4F_preferences_t; @@ -120,7 +121,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf /********************************** * Advanced compression functions - * *********************************/ + * ********************************/ typedef void* LZ4F_compressionContext_t; @@ -159,6 +160,7 @@ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesP /* LZ4F_compressBound() : * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. * preferencesPtr is optional : you can provide NULL as argument, all preferences will then be set to default. + * Note that different preferences will produce in different results. */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr); @@ -207,19 +209,22 @@ typedef struct { /* Resource management */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned version); -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext); +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* ctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t ctx); /* LZ4F_createDecompressionContext() : * The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. * This is achieved using LZ4F_createDecompressionContext(). - * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. * Object can release its memory using LZ4F_freeDecompressionContext(); */ /* Decompression */ -size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); +size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); /* LZ4F_getFrameInfo() * This function decodes frame header information, such as blockSize. * It is optional : you could start by calling directly LZ4F_decompress() instead. @@ -231,7 +236,10 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_ * or an error code which can be tested using LZ4F_isError(). */ -size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* dstBuffer, size_t* dstSizePtr, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr); +size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* optionsPtr); /* LZ4F_decompress() * Call this function repetitively to regenerate data compressed within srcBuffer. * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. @@ -248,8 +256,8 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, void* d * * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. * Basically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it allows less buffer shuffling. - * Note that this is just a hint, you can always provide any srcSize you want. + * Respecting the hint provides some boost to performance, since it does not need intermediate buffers. + * This is just a hint, you can always provide any srcSize you want. * When a frame is fully decoded, the function result will be 0. * If decompression failed, function result is an error code which can be tested using LZ4F_isError(). */ -- cgit v0.12 From 118296aeb0f02b8978544793ae4cc897d7638765 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 2 Dec 2014 23:57:15 +0100 Subject: variable ref renamed --- lib/lz4.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index fb84955..4d3e67f 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -465,7 +465,7 @@ static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableTy { switch (tableType) { - case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } @@ -550,7 +550,7 @@ static int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - const BYTE* ref; + const BYTE* match; BYTE* token; { const BYTE* forwardIp = ip; @@ -566,10 +566,10 @@ static int LZ4_compress_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - ref = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, ctx, tableType, base); if (dict==usingExtDict) { - if (ref<(const BYTE*)source) + if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; @@ -583,13 +583,13 @@ static int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx, tableType, base); - } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip)) - || (LZ4_read32(ref+refDelta) != LZ4_read32(ip)) ); + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); } /* Catch up */ - while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; } + while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } { /* Encode Literal length */ @@ -613,7 +613,7 @@ static int LZ4_compress_generic( _next_match: /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-ref)); op+=2; + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; /* Encode MatchLength */ { @@ -622,10 +622,10 @@ _next_match: if ((dict==usingExtDict) && (lowLimit==dictionary)) { const BYTE* limit; - ref += refDelta; - limit = ip + (dictEnd-ref); + match += refDelta; + limit = ip + (dictEnd-match); if (limit > matchlimit) limit = matchlimit; - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit); + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchLength; if (ip==limit) { @@ -636,7 +636,7 @@ _next_match: } else { - matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit); + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchLength; } @@ -662,10 +662,10 @@ _next_match: LZ4_putPosition(ip-2, ctx, tableType, base); /* Test next position */ - ref = LZ4_getPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, ctx, tableType, base); if (dict==usingExtDict) { - if (ref<(const BYTE*)source) + if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; @@ -677,9 +677,9 @@ _next_match: } } LZ4_putPosition(ip, ctx, tableType, base); - if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1) - && (ref+MAX_DISTANCE>=ip) - && (LZ4_read32(ref+refDelta)==LZ4_read32(ip)) ) + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ -- cgit v0.12 From 65ee6b09c4392814167e7a039d9312aabdb5119b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 3 Dec 2014 19:17:10 +0100 Subject: Fixed : deprecated function LZ4_slideInputBufferHC() --- lib/Makefile | 1 - lib/lz4.c | 4 +-- lib/lz4hc.c | 98 +++++++++++++++++++++++++----------------------------------- 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index f2d585f..3ae12fa 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -63,7 +63,6 @@ else endif default: liblz4 - @cd $(PRGDIR); $(MAKE) -e all: liblz4 diff --git a/lib/lz4.c b/lib/lz4.c index 4d3e67f..f8186b4 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -394,17 +394,17 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi pIn += LZ4_NbCommonBytes(diff); return (unsigned)(pIn - pStart); } + if (LZ4_64bits()) if ((pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } if ((pInhashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; - hc4->base = base - 64 KB; - hc4->inputBuffer = base; - hc4->end = base; - hc4->dictBase = base - 64 KB; + hc4->base = start - 64 KB; + hc4->inputBuffer = start; + hc4->end = start; + hc4->dictBase = start - 64 KB; hc4->dictLimit = 64 KB; hc4->lowLimit = 64 KB; } @@ -155,21 +155,6 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Note : need to handle risk of index overflow - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, @@ -571,7 +556,7 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, /***************************** - Using external allocation + Using external allocation *****************************/ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } @@ -598,9 +583,10 @@ int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, c { return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } + /************************************** - Experimental Streaming Functions -**************************************/ + * Streaming Functions + * ************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }; @@ -616,55 +602,68 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; } - LZ4HC_init (streamPtr, (const BYTE*)dictionary); - if (dictSize >= 4) LZ4HC_Insert (streamPtr, (const BYTE*)dictionary +(dictSize-3)); - streamPtr->end = (const BYTE*)dictionary + dictSize; + LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + ctxPtr->end = (const BYTE*)dictionary + dictSize; return dictSize; } /* compression */ -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { /* auto-init if forgotten */ - if (dsPtr->base == NULL) - LZ4HC_init (dsPtr, (const BYTE*) source); + if (ctxPtr->base == NULL) + LZ4HC_init (ctxPtr, (const BYTE*) source); /* Check overflow */ - if ((size_t)(dsPtr->end - dsPtr->base) > 2 GB) + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(dsPtr->end - dsPtr->base) - dsPtr->dictLimit; + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC((LZ4_streamHC_t*)dsPtr, (const char*)(dsPtr->end) - dictSize, (int)dictSize); + LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ - if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); + if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; - const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; + const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; - dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); - if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; } } - return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); + return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); } int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) @@ -688,7 +687,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); - //LZ4_loadDictHC(LZ4_streamHCPtr, safeBuffer, dictSize); { U32 endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; @@ -749,20 +747,6 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer; - - if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */ - - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); + int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); + return (char*)(hc4->inputBuffer + dictSize); } -- cgit v0.12 From b827ecf72894156c1727482b145dc9e7a116dbc9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 3 Dec 2014 23:19:11 +0100 Subject: Clarified a few comments --- Makefile | 7 +++--- NEWS | 1 + lib/lz4.c | 8 +++---- lib/lz4.h | 9 ++++++-- lib/lz4hc.c | 4 ++-- lib/lz4hc.h | 71 ++++++++++--------------------------------------------------- 6 files changed, 28 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index 66c8c4b..f5bf4b5 100644 --- a/Makefile +++ b/Makefile @@ -72,12 +72,13 @@ endif default: lz4programs - @cd $(PRGDIR); $(MAKE) -e -all: lz4programs +all: + @cd $(LZ4DIR); $(MAKE) -e all + @cd $(PRGDIR); $(MAKE) -e all lz4programs: - @cd $(PRGDIR); $(MAKE) -e all + @cd $(PRGDIR); $(MAKE) -e clean: @rm -f $(DISTRIBNAME) *.sha1 diff --git a/NEWS b/NEWS index f49535d..45e1871 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ Changed : endian and alignment code Changed : directory structure : new "lib" directory Updated : lz4io, now uses lz4frame Fixed : some alignment warnings under clang +Fixed : deprecated function LZ4_slideInputBufferHC() r124: New : LZ4 HC streaming mode diff --git a/lib/lz4.c b/lib/lz4.c index f8186b4..f70ac46 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -1316,11 +1316,9 @@ void* LZ4_create (const char* inputBuffer) char* LZ4_slideInputBuffer (void* LZ4_Data) { - LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data; - - LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB); - - return (char*)(lz4ds->bufferStart + 64 KB); + LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + int dictSize = LZ4_saveDict((LZ4_stream_t*)ctx, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); } /* Obsolete compresson functions using User-allocated state */ diff --git a/lib/lz4.h b/lib/lz4.h index 22bbcb5..7392320 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -178,6 +178,8 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS * LZ4_stream_t * information structure to track an LZ4 stream. * important : init this structure content before first use ! + * note : only allocated directly the structure if you are statically linking LZ4 + * If you are using liblz4 as a DLL, please use below construction methods instead. */ typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; @@ -188,9 +190,10 @@ typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); /* - * If you prefer dynamic allocation methods, * LZ4_createStream will allocate and initialize an LZ4_stream_t structure * LZ4_freeStream releases its memory. + * In the context of a DLL (liblz4), please use these methods rather than the static struct. + * They are more future proof, in case of a change of LZ4_stream_t size. */ LZ4_stream_t* LZ4_createStream(void); int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); @@ -241,7 +244,9 @@ typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_strea * LZ4_streamDecode_t * information structure to track an LZ4 stream. * init this structure content using LZ4_setStreamDecode or memset() before first use ! - * If you prefer dynamic allocation methods : + * + * In the context of a DLL (liblz4) please prefer usage of construction methods below. + * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure * LZ4_freeStreamDecode releases its memory. */ diff --git a/lib/lz4hc.c b/lib/lz4hc.c index c3e0b6d..6690e81 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -556,8 +556,8 @@ int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, /***************************** - Using external allocation -*****************************/ + * Using external allocation + * ***************************/ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 26ba6d3..ce813ab 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -108,21 +108,24 @@ They just use the externally allocated memory for state instead of allocating th #define LZ4_STREAMHCSIZE_U64 32774 #define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U64 * sizeof(unsigned long long)) typedef struct { unsigned long long table[LZ4_STREAMHCSIZE_U64]; } LZ4_streamHC_t; - /* +LZ4_streamHC_t This structure allows static allocation of LZ4 HC streaming state. State must then be initialized using LZ4_resetStreamHC() before first use. -If you prefer dynamic allocation, please refer to functions below. +Static allocation should only be used with statically linked library. +If you want to use LZ4 as a DLL, please use construction functions below, which are more future-proof. */ + LZ4_streamHC_t* LZ4_createStreamHC(void); int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); - /* These functions create and release memory for LZ4 HC streaming state. Newly created states are already initialized. Existing state space can be re-used anytime using LZ4_resetStreamHC(). +If you use LZ4 as a DLL, please use these functions instead of direct struct allocation, +to avoid size mismatch between different versions. */ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); @@ -152,70 +155,18 @@ using LZ4_saveDictHC(). /************************************** - Deprecated Streaming Functions -**************************************/ -/* Note : these streaming functions still follows the older model */ + * Deprecated Streaming Functions + * ************************************/ +/* Note : these streaming functions follows the older model, and should no longer be used */ void* LZ4_createHC (const char* inputBuffer); -//int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize); -//int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize); char* LZ4_slideInputBufferHC (void* LZ4HC_Data); int LZ4_freeHC (void* LZ4HC_Data); int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -/* -These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks. -In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function : - -void* LZ4_createHC (const char* inputBuffer); -The result of the function is the (void*) pointer on the LZ4HC Data Structure. -This pointer will be needed in all other functions. -If the pointer returned is NULL, then the allocation has failed, and compression must be aborted. -The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'. -To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(). -Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), -but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one. -If next block does not begin immediately after the previous one, the compression will fail (return 0). - -When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : -char* LZ4_slideInputBufferHC(void* LZ4HC_Data); -must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer. -Note that, for this function to work properly, minimum size of an input buffer must be 192KB. -==> The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. -*/ - -int LZ4_sizeofStreamStateHC(void); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); - -/* -These functions achieve the same result as : -void* LZ4_createHC (const char* inputBuffer); - -They are provided here to allow the user program to allocate memory using its own routines. - -To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); -Note also that space must be aligned for pointers (32 or 64 bits). - -Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned for pointers (32 or 64 bits), and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamStateHC() must be 0 is OK. -Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). -*/ +int LZ4_sizeofStreamStateHC(void); +int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); #if defined (__cplusplus) -- cgit v0.12