diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-05-20 22:26:03 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-05-20 22:26:03 (GMT) |
commit | 8c38ddd7e67354aa3205ac8c03e5b8f78f567a48 (patch) | |
tree | e6599dd4fb5ca44d2c852154f489725f209f7af9 | |
parent | 9ac680972a3c124e2e512f7202889c4d397c6f9b (diff) | |
download | lz4-8c38ddd7e67354aa3205ac8c03e5b8f78f567a48.zip lz4-8c38ddd7e67354aa3205ac8c03e5b8f78f567a48.tar.gz lz4-8c38ddd7e67354aa3205ac8c03e5b8f78f567a48.tar.bz2 |
Introduce : LZ4_compress_limitedOutput_usingDict()
-rwxr-xr-x | lz4.c | 47 | ||||
-rw-r--r-- | lz4.h | 10 | ||||
-rwxr-xr-x | programs/fullbench.c | 16 |
3 files changed, 48 insertions, 25 deletions
@@ -250,7 +250,7 @@ typedef struct { U32 dictSize; } LZ4_dict_t_internal; -typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive; +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; typedef enum { byPtr, byU32, byU16 } tableType_t; typedef enum { noDict = 0, withPrefix64k = 1, usingDict = 2 } dict_directive; @@ -425,7 +425,7 @@ int LZ4_compress_generic( int inputSize, int maxOutputSize, - limitedOutput_directive limitedOutput, + limitedOutput_directive outputLimited, tableType_t tableType, dict_directive dict) { @@ -495,7 +495,7 @@ int LZ4_compress_generic( /* Encode Literal length */ unsigned litLength = (unsigned)(ip - anchor); token = op++; - if ((limitedOutput) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > oend))) return 0; /* Check output limit */ + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > oend))) return 0; /* Check output limit */ if (litLength>=RUN_MASK) { int len = (int)litLength-RUN_MASK; @@ -519,7 +519,7 @@ _next_match: ip += matchLength + MINMATCH; if (matchLength>=ML_MASK) { - if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > oend))) return 0; /* Check output limit */ + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > oend))) return 0; /* Check output limit */ *token += ML_MASK; matchLength -= ML_MASK; for (; matchLength > 509 ; matchLength-=510) { *op++ = 255; *op++ = 255; } @@ -550,7 +550,7 @@ _last_literals: /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<<ML_BITS); memcpy(op, anchor, iend - anchor); @@ -592,9 +592,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in int result; if (inputSize < (int)LZ4_64KLIMIT) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noDict); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict); else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict); + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict); #if (HEAPMODE) FREEMEM(ctx); @@ -628,15 +628,15 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* MEM_INIT(state, 0, LZ4_sizeofState()); if (inputSize < (int)LZ4_64KLIMIT) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, byU16, noDict); + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict); else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noDict); + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict); } -/*************************************** +/***************************************** Experimental : Streaming functions -***************************************/ +*****************************************/ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source) { @@ -656,11 +656,11 @@ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const char* source) } -int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize) +static int LZ4_compress_usingDict_generic (LZ4_dict_t* LZ4_dict, + const char* source, char* dest, int inputSize, int maxOutputSize, + limitedOutput_directive outputLimited) { LZ4_dict_t_internal* const streamPtr = (LZ4_dict_t_internal*)LZ4_dict; - const int maxOutputSize = 0; - const limitedOutput_directive limitedOutput = notLimited; const tableType_t tableType = byU32; U32 currentOffset; const U32 dictSize = streamPtr->dictSize; @@ -742,7 +742,7 @@ int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest unsigned litLength = (unsigned)(ip - anchor); token = op++; - if ((limitedOutput) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > oend))) return 0; /* Check output limit */ + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > oend))) return 0; /* Check output limit */ if (litLength>=RUN_MASK) { unsigned remaininglength = litLength - RUN_MASK; @@ -775,7 +775,7 @@ int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest ip += matchLength + MINMATCH; if (matchLength>=ML_MASK) { - if ((limitedOutput) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > oend))) return 0; /* Check output limit */ + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > oend))) return 0; /* Check output limit */ *token += ML_MASK; matchLength -= ML_MASK; for (; matchLength > 509 ; matchLength-=510) { *op++ = 255; *op++ = 255; } @@ -798,7 +798,7 @@ int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); - if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<<ML_BITS); memcpy(op, anchor, iend - anchor); @@ -811,6 +811,17 @@ int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest } +int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, 0, notLimited); +} + +int LZ4_compress_limitedOutput_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_usingDict_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput); +} + + int LZ4_setDictPos (LZ4_dict_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_dict_t_internal* dict = (LZ4_dict_t_internal*) LZ4_dict; @@ -1150,5 +1161,5 @@ int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int i int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize) { - return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix64k); + return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k); } @@ -181,11 +181,17 @@ typedef struct { unsigned int table[LZ4_DICTSIZE_U32]; } LZ4_dict_t; /* * LZ4_compress_usingDict - * Compress data block 'source', using blocks compressed before (with the same function) to improve compression ratio + * Compress data block 'source', using blocks compressed before to improve compression ratio * Previous data blocks are assumed to still be present at their previous location. */ int LZ4_compress_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize); -//int LZ4_compress_limitedOutput_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* + * LZ4_compress_limitedOutput_usingDict + * Same as before, but also specify a maximum target compressed size (maxOutputSize) + * If it cannot be met, compression exits, and return a zero. + */ +int LZ4_compress_limitedOutput_usingDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize, int maxOutputSize); /* * LZ4_setDictPos diff --git a/programs/fullbench.c b/programs/fullbench.c index fe13d0b..cc39b15 100755 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -295,6 +295,11 @@ static int local_LZ4_compress_usingDict(const char* in, char* out, int inSize) return LZ4_compress_usingDict(&LZ4_dict, in, out, inSize); } +static int local_LZ4_compress_limitedOutput_usingDict(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_usingDict(&LZ4_dict, in, out, inSize, LZ4_compressBound(inSize)); +} + static void* stateLZ4HC; static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) @@ -359,17 +364,17 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 13 +# define NB_COMPRESSION_ALGORITHMS 14 # define MINCOMPRESSIONCHAR '0' # define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) - double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0}; - double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0}; + double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; + double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; # define NB_DECOMPRESSION_ALGORITHMS 7 # define MINDECOMPRESSIONCHAR '0' # define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS) static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_fast_usingDict", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_usingDict", "LZ4_decompress_safe_partial" }; - double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0}; + double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0}; U64 totals = 0; @@ -498,6 +503,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break; + case 14: compressionFunction = local_LZ4_compress_limitedOutput_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_limitedOutput_usingDict"; break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; } @@ -681,7 +687,7 @@ int main(int argc, char** argv) char* input_filename=0; // Welcome message - DISPLAY( WELCOME_MESSAGE); + DISPLAY(WELCOME_MESSAGE); if (argc<2) { badusage(exename); return 1; } |