From 598bde9a6914287686693013424ec4edf0433992 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 9 Jun 2014 02:42:39 +0100 Subject: converge towards LZ4_compress_continue() --- lz4.c | 53 ---------------------------------------------------- lz4.h | 22 +++++++++++----------- programs/fullbench.c | 38 ++----------------------------------- programs/fuzzer.c | 40 +++++++++++++++++++++++++-------------- 4 files changed, 39 insertions(+), 114 deletions(-) diff --git a/lz4.c b/lz4.c index 2b37c69..6eff9b9 100755 --- a/lz4.c +++ b/lz4.c @@ -751,59 +751,6 @@ void LZ4_renormDictT(LZ4_dict_t_internal* LZ4_dict, const BYTE* src) } -int LZ4_compress_usingDict (void* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest); - - if (dictEnd == (const BYTE*)source) - { - int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - { - int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - -int LZ4_compress_limitedOutput_usingDict (void* LZ4_dict, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_dict; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT((LZ4_dict_t_internal*)LZ4_dict, smallest); - - if (dictEnd == (const BYTE*)source) - { - int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - { - int result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - - int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize) { LZ4_dict_t_internal* streamPtr = (LZ4_dict_t_internal*)LZ4_stream; diff --git a/lz4.h b/lz4.h index 7e38a54..5afc133 100644 --- a/lz4.h +++ b/lz4.h @@ -183,6 +183,7 @@ typedef struct { unsigned int table[LZ4_DICTSIZE_U32]; } LZ4_dict_t; /* * LZ4_createStream * provides a pointer (void*) towards an initialized LZ4_dict_t structure + * LZ4_free just frees it. */ void* LZ4_createStream(); int LZ4_free (void* LZ4_stream); @@ -190,30 +191,30 @@ int LZ4_free (void* LZ4_stream); /* * LZ4_loadDict * Use this function to load a static dictionary into LZ4_dict. - * You can load a size of 0 to init an LZ4_dict_t structure + * Loading a size of 0 is allowed and init the LZ4_dict_t structure. * Return : 1 if OK, 0 if error */ int LZ4_loadDict (void* LZ4_stream, const char* dictionary, int dictSize); /* - * LZ4_compress_usingDict + * LZ4_compress_continue * Compress data block 'source', using blocks compressed before to improve compression ratio * Previous data blocks are assumed to still be present at their previous location. */ -int LZ4_compress_usingDict (void* LZ4_stream, const char* source, char* dest, int inputSize); +int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize); /* - * LZ4_compress_limitedOutput_usingDict + * LZ4_compress_limitedOutput_continue * Same as before, but also specify a maximum target compressed size (maxOutputSize) * If it cannot be met, compression exits, and return a zero. */ -int LZ4_compress_limitedOutput_usingDict (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize); /* * LZ4_moveDict * If previously compressed data block is not guaranteed to remain at its previous memory location * save it into a safe place (char* safeBuffer) - * before calling again LZ4_compress_usingDict() + * before calling again LZ4_compress_continue() * Return : 1 if OK, 0 if error * Note : any dictSize > 64 KB will be interpreted as 64KB. */ @@ -240,14 +241,15 @@ int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compr int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); + /************************************** Obsolete Functions **************************************/ /* These function names are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +- LZ4_uncompress is the same as LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe */ int LZ4_uncompress (const char* source, char* dest, int outputSize); int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); @@ -256,9 +258,7 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isiz void* LZ4_create (const char* inputBuffer); int LZ4_sizeofStreamState(void); int LZ4_resetStreamState(void* state, const char* inputBuffer); -int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize); -char* LZ4_slideInputBuffer (void* LZ4_Data); +char* LZ4_slideInputBuffer (void* state); #if defined (__cplusplus) diff --git a/programs/fullbench.c b/programs/fullbench.c index 7249387..23ca5b7 100755 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -290,16 +290,6 @@ static void* local_LZ4_resetDictT(const char* fake) return NULL; } -static int local_LZ4_compress_usingDict(const char* in, char* out, int inSize) -{ - return LZ4_compress_usingDict(&LZ4_dict, in, out, inSize); -} - -static int local_LZ4_compress_limitedOutput_usingDict(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_usingDict(&LZ4_dict, in, out, inSize, LZ4_compressBound(inSize)); -} - int LZ4_compress_forceExtDict (LZ4_dict_t* LZ4_dict, const char* source, char* dest, int inputSize); static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) { @@ -370,9 +360,8 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 15 +# define NB_COMPRESSION_ALGORITHMS 13 # define MINCOMPRESSIONCHAR '0' -# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; # define NB_DECOMPRESSION_ALGORITHMS 7 @@ -508,9 +497,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; - case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break; - case 14: compressionFunction = local_LZ4_compress_limitedOutput_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_limitedOutput_usingDict"; break; - case 15: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; + case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; } @@ -630,27 +617,6 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) free(chunkP); } -/* - if (nbFiles > 1) - { - int AlgNb; - - DISPLAY(" ** TOTAL ** : \n"); - for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++) - { - char* cName = compressionNames[AlgNb]; - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue; - DISPLAY("%-23.23s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.); - } - for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++) - { - char* dName = decompressionNames[AlgNb]; - if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue; - DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.); - } - } -*/ - if (BMK_pause) { printf("press enter...\n"); getchar(); } return 0; diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 56208d3..608340c 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -83,11 +83,12 @@ #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -//************************************** -// Local Parameters -//************************************** +/***************************************** + Local Parameters +*****************************************/ static int no_prompt = 0; static char* programName; +static int displayLevel = 2; /********************************************************* @@ -200,12 +201,13 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); } +# define FUZ_DISPLAYTEST { testNb++; ((displayLevel<3) || no_prompt) ? 0 : printf("%2i\b\b", testNb); if (displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* LZ4continue; LZ4_dict_t LZ4dict; U32 crcOrig, crcCheck; + int displayRefresh; // Create compressible test buffer @@ -214,6 +216,15 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + // display refresh rate + switch(displayLevel) + { + case 0: displayRefresh = nbCycles+1; break; + case 1: displayRefresh=FUZ_MAX(1, nbCycles / 100); break; + case 2: displayRefresh=99; break; + default : displayRefresh=1; + } + // move to startCycle for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { @@ -231,14 +242,10 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; int blockContinueCompressedSize; - // note : promptThrottle is throtting stdout to prevent - // Travis-CI's output limit (10MB) and false hangup detection. - const int step = FUZ_MAX(1, nbCycles / 100); - const int promptThrottle = ((cycleNb % step) == 0); - if (!no_prompt || cycleNb == 0 || promptThrottle) + if ((cycleNb%displayRefresh) == 0) { printf("\r%7i /%7i - ", cycleNb, nbCycles); - if (no_prompt) fflush(stdout); + fflush(stdout); } // Select block to test @@ -431,19 +438,19 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; memset(&LZ4dict, 0, sizeof(LZ4_dict_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); - blockContinueCompressedSize = LZ4_compress_usingDict(&LZ4dict, block, compressedBuffer, blockSize); + blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed"); FUZ_DISPLAYTEST; memset(&LZ4dict, 0, sizeof(LZ4_dict_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_usingDict(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer"); FUZ_DISPLAYTEST; memset(&LZ4dict, 0, sizeof(LZ4_dict_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); - ret = LZ4_compress_limitedOutput_usingDict(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); + ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer"); // Decompress with dictionary as external @@ -535,6 +542,7 @@ int FUZ_usage() DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); DISPLAY( " -h : display help and exit\n"); return 0; } @@ -561,7 +569,7 @@ int main(int argc, char** argv) { // Decode command (note : aggregated commands are allowed) if (argument[0]=='-') { - if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; continue; } + if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; } while (argument[1]!=0) { @@ -570,6 +578,10 @@ int main(int argc, char** argv) { { case 'h': return FUZ_usage(); + case 'v': + argument++; + displayLevel=4; + break; case 'i': argument++; nbTests=0; -- cgit v0.12