diff options
author | Przemyslaw Skibinski <inikep@gmail.com> | 2016-11-14 10:29:11 (GMT) |
---|---|---|
committer | Przemyslaw Skibinski <inikep@gmail.com> | 2016-11-14 10:29:11 (GMT) |
commit | 9ad7508db0cfd5473ae722deab981cc9bbcafb79 (patch) | |
tree | 3630ac4568271cf979573c5ad1badbc22406ca47 /lib | |
parent | ec6fb477b283b88feeb7173bd3d87faad16cf0e7 (diff) | |
parent | ecc55d19ba7ce083899459ebbe4b14c109e2bea9 (diff) | |
download | lz4-9ad7508db0cfd5473ae722deab981cc9bbcafb79.zip lz4-9ad7508db0cfd5473ae722deab981cc9bbcafb79.tar.gz lz4-9ad7508db0cfd5473ae722deab981cc9bbcafb79.tar.bz2 |
Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev
Diffstat (limited to 'lib')
-rw-r--r-- | lib/lz4.c | 132 | ||||
-rw-r--r-- | lib/lz4.h | 195 | ||||
-rw-r--r-- | lib/lz4frame.c | 108 | ||||
-rw-r--r-- | lib/lz4frame.h | 5 | ||||
-rw-r--r-- | lib/lz4frame_static.h | 2 | ||||
-rw-r--r-- | lib/lz4hc.c | 124 | ||||
-rw-r--r-- | lib/lz4hc.h | 234 |
7 files changed, 425 insertions, 375 deletions
@@ -41,7 +41,9 @@ * Select how default compression functions will allocate memory for their hash table, * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). */ -#define HEAPMODE 0 +#ifndef HEAPMODE +# define HEAPMODE 0 +#endif /* * ACCELERATION_DEFAULT : @@ -109,8 +111,7 @@ # endif #endif /* _MSC_VER */ -/* LZ4_GCC_VERSION is defined into lz4.h */ -#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) @@ -299,7 +300,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); return (int)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; @@ -310,7 +311,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; @@ -323,7 +324,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clzll((U64)val) >> 3); # else unsigned r; @@ -337,7 +338,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); -# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clz((U32)val) >> 3); # else unsigned r; @@ -371,10 +372,6 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi /*-************************************ * Local Constants **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1)); static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ @@ -382,15 +379,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /*-************************************ * Local Structures and types **************************************/ -typedef struct { - U32 hashTable[HASH_SIZE_U32]; - U32 currentOffset; - U32 initCheck; - const BYTE* dictionary; - BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ - U32 dictSize; -} LZ4_stream_t_internal; - typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; typedef enum { byPtr, byU32, byU16 } tableType_t; @@ -470,7 +458,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ FORCE_INLINE int LZ4_compress_generic( - void* const ctx, + LZ4_stream_t_internal* const dictPtr, const char* const source, char* const dest, const int inputSize, @@ -481,8 +469,6 @@ FORCE_INLINE int LZ4_compress_generic( const dictIssue_directive dictIssue, const U32 acceleration) { - LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; - const BYTE* ip = (const BYTE*) source; const BYTE* base; const BYTE* lowLimit; @@ -523,7 +509,7 @@ FORCE_INLINE int LZ4_compress_generic( if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* First Byte */ - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -543,7 +529,7 @@ FORCE_INLINE int LZ4_compress_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -553,7 +539,7 @@ FORCE_INLINE int LZ4_compress_generic( lowLimit = (const BYTE*)source; } } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, tableType, base); } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) @@ -626,10 +612,10 @@ _next_match: if (ip > mflimit) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, dictPtr->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -638,7 +624,7 @@ _next_match: refDelta = 0; lowLimit = (const BYTE*)source; } } - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) && (match+MAX_DISTANCE>=ip) && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) @@ -673,19 +659,20 @@ _last_literals: int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; LZ4_resetStream((LZ4_stream_t*)state); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } @@ -722,9 +709,9 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m LZ4_resetStream(&ctx); if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -733,7 +720,7 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m ********************************/ static int LZ4_compress_destSize_generic( - void* const ctx, + LZ4_stream_t_internal* const ctx, const char* const src, char* const dst, int* const srcSizePtr, @@ -765,7 +752,7 @@ static int LZ4_compress_destSize_generic( /* First Byte */ *srcSizePtr = 0; - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -786,9 +773,9 @@ static int LZ4_compress_destSize_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); @@ -847,11 +834,11 @@ _next_match: if (op > oMaxSeq) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); - LZ4_putPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); if ( (match+MAX_DISTANCE>=ip) && (LZ4_read32(match)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -888,17 +875,17 @@ _last_literals: } -static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) { - LZ4_resetStream((LZ4_stream_t*)state); + LZ4_resetStream(state); if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); else - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); } } @@ -906,10 +893,10 @@ static int LZ4_compress_destSize_extState (void* state, const char* src, char* d int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctxBody; - void* ctx = &ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); @@ -949,7 +936,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) #define HASH_UNIT sizeof(size_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; @@ -987,7 +974,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - for (i=0; i<HASH_SIZE_U32; i++) { + for (i=0; i<LZ4_HASH_SIZE_U32; i++) { if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } @@ -1000,7 +987,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = (const BYTE*) source; @@ -1023,9 +1010,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch if (dictEnd == (const BYTE*)source) { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; @@ -1034,9 +1021,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* external dictionary mode */ { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -1048,15 +1035,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = dictEnd; if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + LZ4_renormDictT(streamPtr, smallest); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; @@ -1075,7 +1062,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* const dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ @@ -1280,13 +1267,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -typedef struct { - const BYTE* externalDict; - size_t extDictSize; - const BYTE* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - /* * If you prefer dynamic allocation methods, * LZ4_createStreamDecode() @@ -1313,7 +1293,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; lz4sd->prefixSize = (size_t) dictSize; lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; @@ -1330,7 +1310,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1356,7 +1336,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1442,29 +1422,29 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) { - MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); - lz4ds->bufferStart = base; + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; } int LZ4_resetStreamState(void* state, char* inputBuffer) { if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); return 0; } void* LZ4_create (char* inputBuffer) { - void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); return lz4ds; } char* LZ4_slideInputBuffer (void* LZ4_Data) { - LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); return (char*)(ctx->bufferStart + dictSize); } @@ -39,6 +39,10 @@ extern "C" { #endif +/* --- Dependency --- */ +#include <stddef.h> /* size_t */ + + /** Introduction @@ -53,10 +57,12 @@ extern "C" { - unbounded multiple steps (described as Streaming compression) lz4.h provides block compression functions. It gives full buffer control to user. - Block compression functions are not-enough to send information, - since it's still necessary to provide metadata (such as compressed size), - and each application can do it in whichever way it wants. - For interoperability, there is LZ4 frame specification (doc/lz4_Frame_format.md). + Decompressing an lz4-compressed block also requires metadata (such as compressed size). + Each application is free to encode such metadata in whichever way it wants. + + An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), + take care of encoding standard metadata alongside LZ4-compressed blocks. + If your application requires interoperability, it's recommended to use it. A library is provided to take care of it, see lz4frame.h. */ @@ -77,9 +83,6 @@ extern "C" { /*========== Version =========== */ -LZ4LIB_API int LZ4_versionNumber (void); -LZ4LIB_API const char* LZ4_versionString (void); - #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ @@ -91,6 +94,9 @@ LZ4LIB_API const char* LZ4_versionString (void); #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) +LZ4LIB_API int LZ4_versionNumber (void); +LZ4LIB_API const char* LZ4_versionString (void); + /*-************************************ * Tuning parameter @@ -220,31 +226,21 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int /*-********************************************* * Streaming Compression Functions ***********************************************/ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * important : init this structure content before first use ! - * note : only allocated directly the structure if you are statically linking LZ4 - * If you are using liblz4 as a DLL, please use below construction methods instead. - */ -typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; - -/*! LZ4_resetStream() : - * Use this function to init an allocated `LZ4_stream_t` structure - */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +typedef struct LZ4_stream_s LZ4_stream_t; /* incomplete type (defined later) */ /*! LZ4_createStream() and LZ4_freeStream() : * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. * LZ4_freeStream() releases its memory. - * In the context of a DLL (liblz4), please use these methods rather than the static struct. - * They are more future proof, in case of a change of `LZ4_stream_t` size. */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +/*! LZ4_resetStream() : + * An LZ4_stream_t structure can be allocated once and re-used multiple times. + * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + /*! LZ4_loadDict() : * Use this function to load a static dictionary into LZ4_stream. * Any previous data will be forgotten, only 'dictionary' will remain in memory. @@ -274,21 +270,11 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict /*-********************************************** * Streaming Decompression Functions +* Bufferless synchronous API ************************************************/ +typedef struct LZ4_streamDecode_s LZ4_streamDecode_t; /* incomplete type (defined later) */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; -/*! - * LZ4_streamDecode_t - * information structure to track an LZ4 stream. - * init this structure content using LZ4_setStreamDecode or memset() before first use ! - * - * In the context of a DLL (liblz4) please prefer usage of construction methods below. - * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. - * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure - * LZ4_freeStreamDecode releases its memory. - */ +/* creation / destruction of streaming decompression tracking structure */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); @@ -300,7 +286,7 @@ LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_str LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /*! -*_continue() : +LZ4_decompress_*_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) In the case of a ring buffers, decoding buffer must be either : @@ -320,50 +306,135 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); -/*! -*_usingDict() : -Advanced decoding functions : - These decoding functions work the same as - a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() - They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. -*/ +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ +/*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Using these definitions will expose code to API and/or ABI break in future versions of the library. + **************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#endif + +/*! + * LZ4_stream_t : + * information structure to track an LZ4 stream. + * init this structure before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +struct LZ4_stream_s { + union { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; + }; +} ; /* previously typedef'd to LZ4_stream_t */ + + +/*! + * LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode (or memset()) before first use + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +struct LZ4_streamDecode_s { + union { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; + }; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + /*=************************************ * Obsolete Functions **************************************/ -/* Deprecate Warnings */ -/* Should these warnings messages be a problem, +/* Deprecation warnings */ +/* Should these warnings be a problem, it is generally possible to disable them, - with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual for example. - Otherwise, you can also define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED() /* disable deprecation warnings */ +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else -# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) +# elif defined(__GNUC__) && (__GNUC__ >= 3) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# warning "WARNING: You need to implement LZ4_DEPRECATED for this compiler" # define LZ4_DEPRECATED(message) # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -/* These functions will generate warnings in a future release */ -LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ /* These function names are completely deprecated and must no longer be used. diff --git a/lib/lz4frame.c b/lib/lz4frame.c index f5b62dc..c31f82d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; +static const size_t maxFHSize = 15; /* max Frame Header Size */ static const size_t BHSize = 4; @@ -201,10 +201,16 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) return codeError; } +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) +{ + if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError; + return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); +} + static LZ4F_errorCode_t err0r(LZ4F_errorCodes code) { LZ4_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ - return (LZ4F_errorCode_t)-(ptrdiff_t)code; + return (LZ4F_errorCode_t)-(ptrdiff_t)code; } unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } @@ -213,6 +219,8 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } /*-************************************ * Private functions **************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + static size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; @@ -223,7 +231,6 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) return blockSizes[blockSizeID]; } - static BYTE LZ4F_headerChecksum (const void* header, size_t length) { U32 const xxh = XXH32(header, length, 0); @@ -232,14 +239,13 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length) /*-************************************ -* Simple compression functions +* Simple-pass compression functions **************************************/ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) { LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; size_t maxBlockSize = 64 KB; - while (requestedBSID > proposedBSID) - { + while (requestedBSID > proposedBSID) { if (srcSize <= maxBlockSize) return proposedBSID; proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); @@ -248,23 +254,39 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI return requestedBSID; } +static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) +{ + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(bid); + size_t const maxBuffered = blockSize - 1; + size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); + size_t const maxSrcSize = srcSize + bufferedSize; + unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); + size_t const partialBlockSize = srcSize & (blockSize-1); + size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; + unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); + + size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ + size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockHeaderSize * nbBlocks) + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;; + } +} size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefs; - size_t headerSize; - size_t streamSize; + size_t const headerSize = maxFHSize; /* max header size, including magic number and frame content size */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; else memset(&prefs, 0, sizeof(prefs)); - - prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - headerSize = maxFHSize; /* header size, including magic number and frame content size*/ - streamSize = LZ4F_compressBound(srcSize, &prefs); - - return headerSize + streamSize; + return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; } @@ -376,11 +398,11 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. + * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) +size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; BYTE* const dstStart = (BYTE*)dstBuffer; @@ -388,7 +410,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize BYTE* headerStart; size_t requiredBuffSize; - if (dstMaxSize < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); if (cctxPtr->cStage != 0) return err0r(LZ4F_ERROR_GENERIC); memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; @@ -456,25 +478,14 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize } -/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. -* The LZ4F_frameInfo_t structure is optional : -* you can provide NULL as argument, preferences will then be set to cover worst case situations. -* */ +/* LZ4F_compressBound() : + * @ return size of Dst buffer given a srcSize to handle worst case situations. + * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations. + * This function cannot fail. + */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefsNull; - memset(&prefsNull, 0, sizeof(prefsNull)); - prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ - { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; - size_t const blockSize = LZ4F_getBlockSize(bid); - unsigned const nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t const lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t const blockInfo = 4; /* default, without block CRC option */ - size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - - return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; - } + return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); } @@ -482,29 +493,29 @@ typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) { - /* compress one block */ + /* compress a single block */ BYTE* const cSizePtr = (BYTE*)dst; U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); LZ4F_writeLE32(cSizePtr, cSize); if (cSize == 0) { /* compression failed */ cSize = (U32)srcSize; - LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); + LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+4, src, srcSize); } return cSize + 4; } -static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) { (void) level; - return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); + return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, 1); } -static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level) { (void) level; - return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, 1); } static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) @@ -534,14 +545,14 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; /*! LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* The most important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) -* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() +* You can get the minimum value of dstCapacity by using LZ4F_compressBound() * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; size_t const blockSize = cctxPtr->maxBlockSize; @@ -550,17 +561,14 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSiz BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; LZ4F_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t compress; + compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - /* select compression function */ - compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; @@ -640,7 +648,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSiz * The function outputs an error code if it fails (can be tested using LZ4F_isError()) * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. */ -size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr) { BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -648,7 +656,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); - if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +8 : block header(4) + block checksum(4) */ + if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */ (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 8fc5d17..e26be76 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -151,7 +151,8 @@ typedef struct { * Simple compression function ***********************************/ /*!LZ4F_compressFrameBound() : - * Returns the maximum possible size of a frame given srcSize content and preferences. + * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. + * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. */ LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); @@ -175,7 +176,7 @@ typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */ typedef struct { - unsigned stableSrc; /* 1 == src content remain present on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ unsigned reserved[3]; } LZ4F_compressOptions_t; diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index 0c154a3..d922e39 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -72,6 +72,8 @@ extern "C" { #endif typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + #if defined (__cplusplus) } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index edc2db0..6de8e8f 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -31,7 +31,7 @@ - LZ4 source repository : https://github.com/lz4/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ - +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ /* ************************************* @@ -44,11 +44,13 @@ * in stack (0:fastest), or in heap (1:requires malloc()). * Since workplace is rather large, heap mode is recommended. */ -#define LZ4HC_HEAPMODE 0 +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 +#endif /* ************************************* -* Includes +* Dependency ***************************************/ #include "lz4hc.h" @@ -75,40 +77,14 @@ /* ************************************* * Local Constants ***************************************/ -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<<DICTIONARY_LOGSIZE) -#define MAXD_MASK (MAXD - 1) - -#define HASH_LOG (DICTIONARY_LOGSIZE-1) -#define HASHTABLESIZE (1 << HASH_LOG) -#define HASH_MASK (HASHTABLESIZE - 1) - #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) /************************************** -* Local Types -**************************************/ -typedef struct -{ - U32 hashTable[HASHTABLESIZE]; - U16 chainTable[MAXD]; - const BYTE* end; /* next block here to continue on current prefix */ - const BYTE* base; /* All index relative to this position */ - const BYTE* dictBase; /* alternate base for extDict */ - BYTE* inputBuffer; /* deprecated */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 compressionLevel; -} LZ4HC_Data_Structure; - - -/************************************** * Local Macros **************************************/ -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] */ /* flexible, MAXD dependent */ +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -118,7 +94,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) /************************************** * HC Compression **************************************/ -static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) +static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); @@ -132,7 +108,7 @@ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) /* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) { U16* const chainTable = hc4->chainTable; U32* const hashTable = hc4->hashTable; @@ -153,7 +129,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } -FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */ +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts) @@ -165,7 +141,6 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I const U32 dictLimit = hc4->dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; - const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; @@ -176,7 +151,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { - match = base + matchIndex; + const BYTE* const match = base + matchIndex; if (*(match+ml) == *(ip+ml) && (LZ4_read32(match) == LZ4_read32(ip))) { @@ -184,7 +159,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I if (mlt > ml) { ml = mlt; *matchpos = match; } } } else { - match = dictBase + matchIndex; + const BYTE* const match = dictBase + matchIndex; if (LZ4_read32(match) == LZ4_read32(ip)) { size_t mlt; const BYTE* vLimit = ip + (dictLimit - matchIndex); @@ -203,7 +178,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( - LZ4HC_Data_Structure* hc4, + LZ4HC_CCtx_internal* hc4, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, @@ -332,7 +307,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( static int LZ4HC_compress_generic ( - void* const ctxvoid, + LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, int const inputSize, @@ -341,7 +316,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; @@ -516,28 +490,29 @@ _Search3: } -int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } +int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { + LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src); + LZ4HC_init (ctx, (const BYTE*)src); if (maxDstSize < LZ4_compressBound(srcSize)) - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); else - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); } int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { -#if LZ4HC_HEAPMODE==1 - LZ4HC_Data_Structure* statePtr = malloc(sizeof(LZ4HC_Data_Structure)); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); #else - LZ4HC_Data_Structure state; - LZ4HC_Data_Structure* const statePtr = &state; + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; #endif - int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); -#if LZ4HC_HEAPMODE==1 + int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif return cSize; @@ -556,14 +531,14 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_st /* initialization */ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL; - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; + LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -577,7 +552,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int /* compression */ -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ @@ -589,10 +564,11 @@ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newB ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, +static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { + LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); @@ -600,7 +576,7 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ @@ -623,9 +599,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { if (maxOutputSize < LZ4_compressBound(inputSize)) - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); else - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); } @@ -633,7 +609,7 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { - LZ4HC_Data_Structure* const streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; @@ -653,8 +629,8 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS /*********************************** * Deprecated Functions ***********************************/ +/* These functions currently generate deprecation warnings */ /* Deprecated compression functions */ -/* These functions are planned to start generate warnings by r131 approximately */ int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } @@ -668,45 +644,41 @@ int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, /* Deprecated streaming functions */ -/* These functions currently generate deprecation warnings */ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { + LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init(ctx, (const BYTE*)inputBuffer); + ctx->inputBuffer = (BYTE*)inputBuffer; return 0; } void* LZ4_createHC (char* inputBuffer) { - void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); + LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ - LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; return hc4; } -int LZ4_freeHC (void* LZ4HC_Data) -{ - FREEMEM(LZ4HC_Data); - return (0); -} +int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); + LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); return (char*)(hc4->inputBuffer + dictSize); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 563ad98..7adc8fa 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2011-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -34,121 +34,85 @@ #ifndef LZ4_HC_H_19834876238432 #define LZ4_HC_H_19834876238432 - #if defined (__cplusplus) extern "C" { #endif -/*-*************************** -* Includes -*****************************/ -#include <stddef.h> /* size_t */ - -/*-*************************************************************** -* Export parameters -*****************************************************************/ -/*! -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4HCLIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4HCLIB_API __declspec(dllimport) -#else -# define LZ4HCLIB_API -#endif +/* --- Dependency --- */ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ +/* --- Useful constants --- */ #define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_DEFAULT_CLEVEL 9 #define LZ4HC_MAX_CLEVEL 16 + /*-************************************ -* Block Compression -**************************************/ -/*! -LZ4_compress_HC() : - Destination buffer `dst` must be already allocated. - Compression success is guaranteed if `dst` buffer is sized to handle worst circumstances (data not compressible) - Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") - `srcSize` : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") - `compressionLevel` : Recommended values are between 4 and 9, although any value between 0 and LZ4HC_MAX_CLEVEL will work. - 0 means "use default value" (see lz4hc.c). - Values >LZ4HC_MAX_CLEVEL behave the same as 16. - @return : the number of bytes written into buffer 'dst' - or 0 if compression fails. -*/ -LZ4HCLIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. + * Values >LZ4HC_MAX_CLEVEL behave the same as 16. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); /* Note : - Decompression functions are provided within "lz4.h" (BSD license) -*/ + * Decompression functions are provided within "lz4.h" (BSD license) + */ -/*! -LZ4_compress_HC_extStateHC() : - Use this function if you prefer to manually allocate memory for compression tables. - To know how much memory must be allocated for the compression tables, use : - int LZ4_sizeofStateHC(); - - Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). - - The allocated memory can then be provided to the compression functions using 'void* state' parameter. - LZ4_compress_HC_extStateHC() is equivalent to previously described function. - It just uses externally allocated memory for stateHC. -*/ -LZ4HCLIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); -LZ4HCLIB_API int LZ4_sizeofStateHC(void); +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + */ +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +LZ4LIB_API int LZ4_sizeofStateHC(void); /*-************************************ -* Streaming Compression -**************************************/ -#define LZ4_STREAMHCSIZE 262192 -#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) -typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; -/* - LZ4_streamHC_t - This structure allows static allocation of LZ4 HC streaming state. - State must then be initialized using LZ4_resetStreamHC() before first use. - - Static allocation should only be used in combination with static linking. - If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof. -*/ - + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef struct LZ4_streamHC_s LZ4_streamHC_t; /* incomplete type (defined later) */ /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : - These functions create and release memory for LZ4 HC streaming state. - Newly created states are already initialized. - Existing state space can be re-used anytime using LZ4_resetStreamHC(). - If you use LZ4 as a DLL, use these functions instead of static structure allocation, - to avoid size mismatch between different versions. -*/ -LZ4HCLIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); -LZ4HCLIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * Existing states can be re-used several times, using LZ4_resetStreamHC(). + * These methods are API and ABI stable, they can be used in combination with a DLL. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); -LZ4HCLIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); -LZ4HCLIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); +LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); -LZ4HCLIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); -LZ4HCLIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); /* These functions compress data in successive blocks of any size, using previous blocks as dictionary. One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. There is an exception for ring buffers, which can be smaller than 64 KB. - Such case is automatically detected and correctly handled by LZ4_compress_HC_continue(). + Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). Then, use LZ4_compress_HC_continue() to compress each successive block. - It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation. + 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success. If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, you must save it to a safer memory space, using LZ4_saveDictHC(). @@ -156,50 +120,102 @@ LZ4HCLIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, */ +/*-****************************************** + * !!!!! STATIC LINKING ONLY !!!!! + *******************************************/ + + /*-************************************* + * PRIVATE DEFINITIONS : + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_streamHC_t`. + * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 + **************************************/ +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) +#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) + +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) +#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) + + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct +{ + uint32_t hashTable[LZ4HC_HASHTABLESIZE]; + uint16_t chainTable[LZ4HC_MAXD]; + const uint8_t* end; /* next block here to continue on current prefix */ + const uint8_t* base; /* All index relative to this position */ + const uint8_t* dictBase; /* alternate base for extDict */ + uint8_t* inputBuffer; /* deprecated */ + uint32_t dictLimit; /* below that point, need extDict */ + uint32_t lowLimit; /* below that point, no more dict */ + uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t compressionLevel; +} LZ4HC_CCtx_internal; + +#else + +typedef struct +{ + unsigned int hashTable[LZ4HC_HASHTABLESIZE]; + unsigned short chainTable[LZ4HC_MAXD]; + const unsigned char* end; /* next block here to continue on current prefix */ + const unsigned char* base; /* All index relative to this position */ + const unsigned char* dictBase; /* alternate base for extDict */ + unsigned char* inputBuffer; /* deprecated */ + unsigned int dictLimit; /* below that point, need extDict */ + unsigned int lowLimit; /* below that point, no more dict */ + unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int compressionLevel; +} LZ4HC_CCtx_internal; + +#endif + +#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +struct LZ4_streamHC_s { + union { + size_t table[LZ4_STREAMHCSIZE_SIZET]; + LZ4HC_CCtx_internal internal_donotuse; + }; +}; /* previously typedef'd to LZ4_streamHC_t */ +/* + LZ4_streamHC_t : + This structure allows static allocation of LZ4 HC streaming state. + State must be initialized using LZ4_resetStreamHC() before first use. + + Static allocation shall only be used in combination with static linking. + When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. +*/ + /*-************************************ * Deprecated Functions **************************************/ -/* Deprecate Warnings */ -/* Should these warnings messages be a problem, - it is generally possible to disable them, - with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual for example. - You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ -#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK -# define LZ4_DEPRECATE_WARNING_DEFBLOCK -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ +/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ /* deprecated compression functions */ /* these functions will trigger warning messages in future releases */ -LZ4HCLIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4HCLIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -LZ4HCLIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -LZ4HCLIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Deprecated Streaming functions using older model; should no longer be used */ LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); |