diff options
| -rw-r--r-- | doc/lz4_manual.html | 68 | ||||
| -rw-r--r-- | lib/lz4.c | 117 | ||||
| -rw-r--r-- | lib/lz4.h | 70 | ||||
| -rw-r--r-- | lib/lz4hc.c | 82 | ||||
| -rw-r--r-- | lib/lz4hc.h | 60 |
5 files changed, 262 insertions, 135 deletions
diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 838dbf4..bc46645 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -10,10 +10,11 @@ <ol> <li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter2">Tuning parameter</a></li> -<li><a href="#Chapter3">Simple Functions</a></li> -<li><a href="#Chapter4">Advanced Functions</a></li> -<li><a href="#Chapter5">Streaming Compression Functions</a></li> -<li><a href="#Chapter6">Streaming Decompression Functions</a></li> +<li><a href="#Chapter3">Private definitions</a></li> +<li><a href="#Chapter4">Simple Functions</a></li> +<li><a href="#Chapter5">Advanced Functions</a></li> +<li><a href="#Chapter6">Streaming Compression Functions</a></li> +<li><a href="#Chapter7">Streaming Decompression Functions</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -48,7 +49,45 @@ const char* LZ4_versionString (void); </p></pre><BR> -<a name="Chapter3"></a><h2>Simple Functions</h2><pre></pre> +<a name="Chapter3"></a><h2>Private definitions</h2><pre> + Do not use these definitions. + They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + If you use these definitions in your code, it will break when you upgrade LZ4 to a new version. +<BR></pre> + +<pre><b>typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + uint32_t dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; </b>/* obsolete, used for slideInputBuffer */<b> + unsigned int dictSize; +} LZ4_stream_t_internal; +</b></pre><BR> +<pre><b>typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; +</b></pre><BR> +<a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre> <pre><b>int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); </b><p> Compresses 'sourceSize' bytes from buffer 'source' @@ -75,7 +114,7 @@ const char* LZ4_versionString (void); It never writes outside output buffer, nor reads outside input buffer. </p></pre><BR> -<a name="Chapter4"></a><h2>Advanced Functions</h2><pre></pre> +<a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre> <pre><b>int LZ4_compressBound(int inputSize); </b><p> Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) @@ -137,9 +176,14 @@ int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets </p></pre><BR> -<a name="Chapter5"></a><h2>Streaming Compression Functions</h2><pre></pre> +<a name="Chapter6"></a><h2>Streaming Compression Functions</h2><pre></pre> -<pre><b>typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; +<pre><b>typedef struct { + union { + long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; + }; +} LZ4_stream_t; </b><p> information structure to track an LZ4 stream. important : init this structure content before first use ! note : only allocated directly the structure if you are statically linking LZ4 @@ -187,9 +231,13 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr); </p></pre><BR> -<a name="Chapter6"></a><h2>Streaming Decompression Functions</h2><pre></pre> +<a name="Chapter7"></a><h2>Streaming Decompression Functions</h2><pre></pre> -<pre><b>typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +<pre><b>typedef struct { + union { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; + }; </b></pre><BR> <pre><b>LZ4_streamDecode_t* LZ4_createStreamDecode(void); int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); @@ -371,10 +371,6 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi /*-************************************ * Local Constants **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - static const int LZ4_64Klimit = ((64 KB) + (MFLIMIT-1)); static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ @@ -382,15 +378,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /*-************************************ * Local Structures and types **************************************/ -typedef struct { - U32 hashTable[HASH_SIZE_U32]; - U32 currentOffset; - U32 initCheck; - const BYTE* dictionary; - BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ - U32 dictSize; -} LZ4_stream_t_internal; - typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; typedef enum { byPtr, byU32, byU16 } tableType_t; @@ -470,7 +457,7 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy /** LZ4_compress_generic() : inlined, to ensure branches are decided at compilation time */ FORCE_INLINE int LZ4_compress_generic( - void* const ctx, + LZ4_stream_t_internal* const dictPtr, const char* const source, char* const dest, const int inputSize, @@ -481,8 +468,6 @@ FORCE_INLINE int LZ4_compress_generic( const dictIssue_directive dictIssue, const U32 acceleration) { - LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; - const BYTE* ip = (const BYTE*) source; const BYTE* base; const BYTE* lowLimit; @@ -523,7 +508,7 @@ FORCE_INLINE int LZ4_compress_generic( if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ /* First Byte */ - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -543,7 +528,7 @@ FORCE_INLINE int LZ4_compress_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -553,7 +538,7 @@ FORCE_INLINE int LZ4_compress_generic( lowLimit = (const BYTE*)source; } } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, dictPtr->hashTable, tableType, base); } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) @@ -626,10 +611,10 @@ _next_match: if (ip > mflimit) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, dictPtr->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, dictPtr->hashTable, tableType, base); if (dict==usingExtDict) { if (match < (const BYTE*)source) { refDelta = dictDelta; @@ -638,7 +623,7 @@ _next_match: refDelta = 0; lowLimit = (const BYTE*)source; } } - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, dictPtr->hashTable, tableType, base); if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) && (match+MAX_DISTANCE>=ip) && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) @@ -673,19 +658,20 @@ _last_literals: int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; LZ4_resetStream((LZ4_stream_t*)state); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; if (maxOutputSize >= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } @@ -722,9 +708,9 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m LZ4_resetStream(&ctx); if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else - return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } @@ -733,7 +719,7 @@ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int m ********************************/ static int LZ4_compress_destSize_generic( - void* const ctx, + LZ4_stream_t_internal* const ctx, const char* const src, char* const dst, int* const srcSizePtr, @@ -765,7 +751,7 @@ static int LZ4_compress_destSize_generic( /* First Byte */ *srcSizePtr = 0; - LZ4_putPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); ip++; forwardH = LZ4_hashPosition(ip, tableType); /* Main Loop */ @@ -786,9 +772,9 @@ static int LZ4_compress_destSize_generic( if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, ctx, tableType, base); + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); @@ -847,11 +833,11 @@ _next_match: if (op > oMaxSeq) break; /* Fill table */ - LZ4_putPosition(ip-2, ctx, tableType, base); + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, ctx, tableType, base); - LZ4_putPosition(ip, ctx, tableType, base); + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); if ( (match+MAX_DISTANCE>=ip) && (LZ4_read32(match)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -888,17 +874,17 @@ _last_literals: } -static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) { - LZ4_resetStream((LZ4_stream_t*)state); + LZ4_resetStream(state); if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); else - return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); } } @@ -906,10 +892,10 @@ static int LZ4_compress_destSize_extState (void* state, const char* src, char* d int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctxBody; - void* ctx = &ctxBody; + LZ4_stream_t* ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); @@ -949,7 +935,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) #define HASH_UNIT sizeof(size_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { - LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; @@ -987,7 +973,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; - for (i=0; i<HASH_SIZE_U32; i++) { + for (i=0; i<LZ4_HASH_SIZE_U32; i++) { if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } @@ -1000,7 +986,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = (const BYTE*) source; @@ -1023,9 +1009,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch if (dictEnd == (const BYTE*)source) { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; @@ -1034,9 +1020,9 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* external dictionary mode */ { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -1048,15 +1034,15 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = dictEnd; if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + LZ4_renormDictT(streamPtr, smallest); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; @@ -1075,7 +1061,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { - LZ4_stream_t_internal* const dict = (LZ4_stream_t_internal*) LZ4_dict; + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ @@ -1280,13 +1266,6 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== streaming decompression functions =====*/ -typedef struct { - const BYTE* externalDict; - size_t extDictSize; - const BYTE* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - /* * If you prefer dynamic allocation methods, * LZ4_createStreamDecode() @@ -1313,7 +1292,7 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; lz4sd->prefixSize = (size_t) dictSize; lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; @@ -1330,7 +1309,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1356,7 +1335,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { - LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { @@ -1442,29 +1421,29 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) { - MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); - lz4ds->bufferStart = base; + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; } int LZ4_resetStreamState(void* state, char* inputBuffer) { if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); return 0; } void* LZ4_create (char* inputBuffer) { - void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); return lz4ds; } char* LZ4_slideInputBuffer (void* LZ4_Data) { - LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); return (char*)(ctx->bufferStart + dictSize); } @@ -39,6 +39,12 @@ extern "C" { #endif +/*^*************************** +* Includes +*****************************/ +#include <stddef.h> /* size_t */ + + /** Introduction @@ -106,6 +112,56 @@ LZ4LIB_API const char* LZ4_versionString (void); /*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * If you use these definitions in your code, it will break when you upgrade LZ4 to a new version. +**************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#endif + +/*-************************************ * Simple Functions **************************************/ /*! LZ4_compress_default() : @@ -229,7 +285,12 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int * note : only allocated directly the structure if you are statically linking LZ4 * If you are using liblz4 as a DLL, please use below construction methods instead. */ -typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; +typedef struct { + union { + long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; + }; +} LZ4_stream_t; /*! LZ4_resetStream() : * Use this function to init an allocated `LZ4_stream_t` structure @@ -278,7 +339,12 @@ LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dict #define LZ4_STREAMDECODESIZE_U64 4 #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +typedef struct { + union { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; + }; +} LZ4_streamDecode_t; /*! * LZ4_streamDecode_t * information structure to track an LZ4 stream. diff --git a/lib/lz4hc.c b/lib/lz4hc.c index edc2db0..825e5bc 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -75,40 +75,14 @@ /* ************************************* * Local Constants ***************************************/ -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<<DICTIONARY_LOGSIZE) -#define MAXD_MASK (MAXD - 1) - -#define HASH_LOG (DICTIONARY_LOGSIZE-1) -#define HASHTABLESIZE (1 << HASH_LOG) -#define HASH_MASK (HASHTABLESIZE - 1) - #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) /************************************** -* Local Types -**************************************/ -typedef struct -{ - U32 hashTable[HASHTABLESIZE]; - U16 chainTable[MAXD]; - const BYTE* end; /* next block here to continue on current prefix */ - const BYTE* base; /* All index relative to this position */ - const BYTE* dictBase; /* alternate base for extDict */ - BYTE* inputBuffer; /* deprecated */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 compressionLevel; -} LZ4HC_Data_Structure; - - -/************************************** * Local Macros **************************************/ -#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -/* #define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] */ /* flexible, MAXD dependent */ +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -332,7 +306,7 @@ FORCE_INLINE int LZ4HC_encodeSequence ( static int LZ4HC_compress_generic ( - void* const ctxvoid, + LZ4HC_Data_Structure* const ctx, const char* const source, char* const dest, int const inputSize, @@ -341,7 +315,6 @@ static int LZ4HC_compress_generic ( limitedOutput_directive limit ) { - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; @@ -520,21 +493,22 @@ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { + LZ4HC_Data_Structure* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src); + LZ4HC_init (ctx, (const BYTE*)src); if (maxDstSize < LZ4_compressBound(srcSize)) - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); else - return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); + return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); } int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { #if LZ4HC_HEAPMODE==1 - LZ4HC_Data_Structure* statePtr = malloc(sizeof(LZ4HC_Data_Structure)); + LZ4_streamHC_t* statePtr = malloc(sizeof(LZ4_streamHC_t)); #else - LZ4HC_Data_Structure state; - LZ4HC_Data_Structure* const statePtr = &state; + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; #endif int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); #if LZ4HC_HEAPMODE==1 @@ -556,14 +530,14 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_st /* initialization */ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL; - ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; + LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { - LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + LZ4HC_Data_Structure* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -589,10 +563,11 @@ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newB ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } -static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, +static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { + LZ4HC_Data_Structure* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); @@ -600,7 +575,7 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; - LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ @@ -623,9 +598,9 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { if (maxOutputSize < LZ4_compressBound(inputSize)) - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); else - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); } @@ -633,7 +608,7 @@ int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* sourc int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { - LZ4HC_Data_Structure* const streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + LZ4HC_Data_Structure* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; @@ -673,18 +648,19 @@ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { + LZ4HC_Data_Structure *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init(ctx, (const BYTE*)inputBuffer); + ctx->inputBuffer = (BYTE*)inputBuffer; return 0; } void* LZ4_createHC (char* inputBuffer) { - void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); + LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ - LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; + LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; return hc4; } @@ -696,17 +672,17 @@ int LZ4_freeHC (void* LZ4HC_Data) int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { - return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; + LZ4HC_Data_Structure* hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); return (char*)(hc4->inputBuffer + dictSize); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 563ad98..0d20e2e 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -64,6 +64,59 @@ extern "C" { #define LZ4HC_DEFAULT_CLEVEL 9 #define LZ4HC_MAX_CLEVEL 16 + +/*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_streamHC_t`. + * If you use these definitions in your code, it will break when you upgrade LZ4 to a new version. +**************************************/ +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) +#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) + +#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) +#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) +#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) + + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include <stdint.h> + +typedef struct +{ + uint32_t hashTable[LZ4HC_HASHTABLESIZE]; + uint16_t chainTable[LZ4HC_MAXD]; + const uint8_t* end; /* next block here to continue on current prefix */ + const uint8_t* base; /* All index relative to this position */ + const uint8_t* dictBase; /* alternate base for extDict */ + uint8_t* inputBuffer; /* deprecated */ + uint32_t dictLimit; /* below that point, need extDict */ + uint32_t lowLimit; /* below that point, no more dict */ + uint32_t nextToUpdate; /* index from which to continue dictionary update */ + uint32_t compressionLevel; +} LZ4HC_Data_Structure; + +#else + +typedef struct +{ + unsigned int hashTable[LZ4HC_HASHTABLESIZE]; + unsigned short chainTable[LZ4HC_MAXD]; + const unsigned char* end; /* next block here to continue on current prefix */ + const unsigned char* base; /* All index relative to this position */ + const unsigned char* dictBase; /* alternate base for extDict */ + unsigned char* inputBuffer; /* deprecated */ + unsigned int dictLimit; /* below that point, need extDict */ + unsigned int lowLimit; /* below that point, no more dict */ + unsigned int nextToUpdate; /* index from which to continue dictionary update */ + unsigned int compressionLevel; +} LZ4HC_Data_Structure; + +#endif + + /*-************************************ * Block Compression **************************************/ @@ -108,7 +161,12 @@ LZ4HCLIB_API int LZ4_sizeofStateHC(void); **************************************/ #define LZ4_STREAMHCSIZE 262192 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) -typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; +typedef struct { + union { + size_t table[LZ4_STREAMHCSIZE_SIZET]; + LZ4HC_Data_Structure internal_donotuse; + }; +} LZ4_streamHC_t; /* LZ4_streamHC_t This structure allows static allocation of LZ4 HC streaming state. |