diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-05-19 23:40:29 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-05-19 23:40:29 (GMT) |
commit | 4db6b03fceac50961a8f127aa2eda73d3373a1fe (patch) | |
tree | f7e81ed5b643e430126fa7a8c82d16e6f3b33799 /programs | |
parent | 7bcb3b2e9f36ad6adef2cb43858a8f3adb39c527 (diff) | |
download | lz4-4db6b03fceac50961a8f127aa2eda73d3373a1fe.zip lz4-4db6b03fceac50961a8f127aa2eda73d3373a1fe.tar.gz lz4-4db6b03fceac50961a8f127aa2eda73d3373a1fe.tar.bz2 |
First version of Block Streaming API : LZ4_compress_usingDict()
Diffstat (limited to 'programs')
-rw-r--r-- | programs/Makefile | 19 | ||||
-rw-r--r-- | programs/fullbench.c | 68 | ||||
-rw-r--r-- | programs/fuzzer.c | 63 | ||||
-rw-r--r-- | programs/lz4.1 | 3 | ||||
-rw-r--r-- | programs/lz4cli.c | 3 |
5 files changed, 97 insertions, 59 deletions
diff --git a/programs/Makefile b/programs/Makefile index c0d6d15..a81a701 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -42,15 +42,6 @@ MANDIR=$(PREFIX)/share/man/man1 LZ4DIR=.. TEST_FILES = COPYING -TEST_TARGETS=test-32 test-64 - -# Minimize test target for Travis CI's Build Matrix -ifeq ($(LZ4_TRAVIS_CI_ENV),-m32) -TEST_TARGETS=test-32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64) -TEST_TARGETS=test-64 -endif - # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -115,11 +106,7 @@ uninstall: [ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1 @echo lz4 successfully uninstalled -test: $(TEST_TARGETS) - -test-32: test-lz4 test-lz4c32 test-fullbench32 test-fuzzer32 - -test-64: test-lz4 test-lz4c test-fullbench test-fuzzer +test: test-lz4 test-lz4c test-lz4c32 test-fullbench test-fullbench32 test-fuzzer test-fuzzer32 test-lz4: @@ -128,10 +115,10 @@ test-lz4c: test-lz4c32: test-fullbench: fullbench - ./fullbench --no-prompt $(TEST_FILES) + ./fullbench --no-prompt -i1 $(TEST_FILES) test-fullbench32: fullbench32 - ./fullbench32 --no-prompt $(TEST_FILES) + ./fullbench32 --no-prompt -i1 $(TEST_FILES) test-fuzzer: fuzzer ./fuzzer --no-prompt diff --git a/programs/fullbench.c b/programs/fullbench.c index 1200ca0..01b807c 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -281,6 +281,21 @@ static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); } + +LZ4_dict_t LZ4_dict; +static inline void* local_LZ4_resetDictT(const char* fake) +{ + (void)fake; + memset(&LZ4_dict, 0, sizeof(LZ4_dict_t)); + return NULL; +} + +static inline int local_LZ4_compress_usingDict(const char* in, char* out, int inSize) +{ + return LZ4_compress_usingDict(&LZ4_dict, in, out, inSize); +} + + static void* stateLZ4HC; static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) { @@ -344,15 +359,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 12 +# define NB_COMPRESSION_ALGORITHMS 13 # define MINCOMPRESSIONCHAR '0' # define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) - static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", - "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState", - "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", - "LZ4_compressHC", "LZ4_compressHC_limitedOutput", - "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC", - "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" }; double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0}; double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0}; # define NB_DECOMPRESSION_ALGORITHMS 7 @@ -465,29 +474,30 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY(" %s : \n", inFileName); // Compression Algorithms - for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) + for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) { - char* cName = compressionNames[cAlgNb]; + char* compressorName; int (*compressionFunction)(const char*, char*, int); void* (*initFunction)(const char*) = NULL; double bestTime = 100000000.; - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb+1)) continue; + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; switch(cAlgNb) { - case 0 : compressionFunction = LZ4_compress; break; - case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break; - case 2 : compressionFunction = local_LZ4_compress_withState; break; - case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break; - case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break; - case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break; - case 6 : compressionFunction = LZ4_compressHC; break; - case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break; - case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break; - case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break; - case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break; - case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break; + case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; + case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; + case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; + case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; + case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break; + case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break; + case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; + case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; + case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; + case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; + case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; + case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; + case 13: compressionFunction = local_LZ4_compress_usingDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_usingDict"; break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; } @@ -496,7 +506,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) double averageTime; int milliTime; - PROGRESS("%1i-%-21.21s : %9i ->\r", loopNb, cName, (int)benchedSize); + PROGRESS("%1i-%-25.25s : %9i ->\r", loopNb, compressorName, (int)benchedSize); { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory nb_loops = 0; @@ -509,7 +519,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1); + if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); } if (initFunction!=NULL) free(ctx); nb_loops++; @@ -520,13 +530,13 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if (averageTime < bestTime) bestTime = averageTime; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; - PROGRESS("%1i-%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + PROGRESS("%1i-%-25.25s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); } if (ratio<100.) - DISPLAY("%-23.23s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + DISPLAY("%-27.27s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); else - DISPLAY("%-23.23s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + DISPLAY("%-27.27s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); totalCTime[cAlgNb] += bestTime; totalCSize[cAlgNb] += cSize; @@ -536,7 +546,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1); + if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); } { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking @@ -607,6 +617,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) free(chunkP); } +/* if (nbFiles > 1) { int AlgNb; @@ -625,6 +636,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("%-31.31s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.); } } +*/ if (BMK_pause) { printf("press enter...\n"); getchar(); } diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 1ea14f6..081f0df 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -187,7 +187,7 @@ int FUZ_SecurityTest() #define FUZ_MAX(a,b) (a>b?a:b) -int FUZ_test(U32 seed, int nbTests, double compressibility) { +int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { unsigned long long bytes = 0; unsigned long long cbytes = 0; unsigned long long hcbytes = 0; @@ -197,10 +197,10 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { char* decodedBuffer; # define FUZ_max LZ4_COMPRESSBOUND(LEN) unsigned int randState=seed; - int ret, attemptNb; + int ret, cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %i : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %i) \n", seed, attemptNb); goto _output_error; } -# define FUZ_DISPLAYTEST testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); + printf(" (seed %u, cycle %i) \n", seed, cycleNb); goto _output_error; } +# define FUZ_DISPLAYTEST { testNb++; no_prompt ? 0 : printf("%2i\b\b", testNb); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); void* LZ4continue; @@ -213,8 +213,16 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { compressedBuffer = malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + // move to startCycle + for (cycleNb = 0; cycleNb < startCycle; cycleNb++) + { + FUZ_rand(&randState); + FUZ_rand(&randState); + FUZ_rand(&randState); + } + // Test loop - for (attemptNb = 0; attemptNb < nbTests; attemptNb++) + for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) { int testNb = 0; char* dict; @@ -224,11 +232,11 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { // note : promptThrottle is throtting stdout to prevent // Travis-CI's output limit (10MB) and false hangup detection. - const int step = FUZ_MAX(1, nbTests / 100); - const int promptThrottle = ((attemptNb % step) == 0); - if (!no_prompt || attemptNb == 0 || promptThrottle) + const int step = FUZ_MAX(1, nbCycles / 100); + const int promptThrottle = ((cycleNb % step) == 0); + if (!no_prompt || cycleNb == 0 || promptThrottle) { - printf("\r%7i /%7i - ", attemptNb, nbTests); + printf("\r%7i /%7i - ", cycleNb, nbCycles); if (no_prompt) fflush(stdout); } @@ -305,7 +313,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space"); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than target size"); // well, is that an issue ? + //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size"); crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); @@ -409,6 +417,18 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); + // Compress using dictionary + FUZ_DISPLAYTEST; + dict -= 9; + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + { + LZ4_dict_t LZ4dict; + memset(&LZ4dict, 0, sizeof(LZ4_dict_t)); + LZ4_loadDict(&LZ4dict, dict, dictSize); + blockContinueCompressedSize = LZ4_compress_usingDict(&LZ4dict, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed"); + } + // Decompress with dictionary as external FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; @@ -416,6 +436,13 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) + { + int i=0; + while (block[i]==decodedBuffer[i]) i++; + printf("Wrong Byte at position %i/%i\n", i, blockSize); + + } FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data"); FUZ_DISPLAYTEST; @@ -455,7 +482,7 @@ int FUZ_test(U32 seed, int nbTests, double compressibility) { ccbytes += blockContinueCompressedSize; } - printf("\r%7i /%7i - ", attemptNb, nbTests); + printf("\r%7i /%7i - ", cycleNb, nbCycles); printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); @@ -489,6 +516,7 @@ int FUZ_usage() DISPLAY( "Arguments :\n"); DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); DISPLAY( " -h : display help and exit\n"); return 0; @@ -502,6 +530,7 @@ int main(int argc, char** argv) { int seedset=0; int argNb; int nbTests = NB_ATTEMPTS; + int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; // Check command line @@ -544,6 +573,16 @@ int main(int argc, char** argv) { argument++; } break; + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) + { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; case 'p': argument++; proba=0; @@ -583,5 +622,5 @@ int main(int argc, char** argv) { if (nbTests<=0) nbTests=1; - return FUZ_test(seed, nbTests, ((double)proba) / 100); + return FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); } diff --git a/programs/lz4.1 b/programs/lz4.1 index 298cbf6..6ae8d3c 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -64,6 +64,7 @@ following options .TP .B \-B# block size [4-7](default : 7) + B4= 64KB ; B5= 256KB ; B6= 1MB ; B7= 4MB .TP .B \-BD block dependency (improve compression ratio) @@ -84,4 +85,4 @@ following options Report bugs at:- https://code.google.com/p/lz4/ .SH AUTHOR -Yann Collet
\ No newline at end of file +Yann Collet diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 934c2bb..1c4e9de 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -405,9 +405,8 @@ int main(int argc, char** argv) case '7': { int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_SetBlocksize(S); blockSize = LZ4IO_setBlockSizeID(B); + BMK_SetBlocksize(blockSize); argument++; break; } |