diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2022-07-30 08:49:25 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-07-30 08:49:25 (GMT) |
commit | 2042692400bc0fe3a9376715db7c33e5f16d83f6 (patch) | |
tree | fd59adcab9524f98757f967880b1fcd492c05215 | |
parent | 5c8bb8739eb5e503a02a1976d1ddcadb049a041d (diff) | |
parent | 9978bb90cf45e455a146a7beaf2e4d08baaed6c0 (diff) | |
download | lz4-2042692400bc0fe3a9376715db7c33e5f16d83f6.zip lz4-2042692400bc0fe3a9376715db7c33e5f16d83f6.tar.gz lz4-2042692400bc0fe3a9376715db7c33e5f16d83f6.tar.bz2 |
Merge pull request #1122 from lz4/skipCrc
Introduce ability to save cpu cycles by disabling checksum validation
-rw-r--r-- | lib/lz4frame.c | 32 | ||||
-rw-r--r-- | lib/lz4frame.h | 8 | ||||
-rw-r--r-- | programs/bench.c | 17 | ||||
-rw-r--r-- | programs/bench.h | 1 | ||||
-rw-r--r-- | programs/lz4.1.md | 3 | ||||
-rw-r--r-- | programs/lz4cli.c | 5 | ||||
-rw-r--r-- | programs/lz4io.c | 29 | ||||
-rw-r--r-- | tests/Makefile | 4 | ||||
-rw-r--r-- | tests/frametest.c | 1 |
9 files changed, 75 insertions, 25 deletions
diff --git a/lib/lz4frame.c b/lib/lz4frame.c index fa987d7..cba5744 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1222,6 +1222,7 @@ struct LZ4F_dctx_s { size_t tmpOutStart; XXH32_state_t xxh; XXH32_state_t blockChecksum; + int skipChecksum; BYTE header[LZ4F_HEADER_SIZE_MAX]; }; /* typedef'd to LZ4F_dctx in lz4frame.h */ @@ -1275,6 +1276,7 @@ void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) dctx->dStage = dstage_getFrameHeader; dctx->dict = NULL; dctx->dictSize = 0; + dctx->skipChecksum = 0; } @@ -1537,7 +1539,6 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx, } - /*! LZ4F_decompress() : * Call this function repetitively to regenerate compressed data in srcBuffer. * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer @@ -1581,6 +1582,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, *srcSizePtr = 0; *dstSizePtr = 0; assert(dctx != NULL); + dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */ /* behaves as a state machine */ @@ -1711,11 +1713,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctx->frameInfo.blockChecksumFlag) { - (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + if (!dctx->skipChecksum) { + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); } - if (dctx->frameInfo.contentChecksumFlag) - (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= sizeToCopy; @@ -1761,7 +1765,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } crcSrc = dctx->header; } - { U32 const readCRC = LZ4F_readLE32(crcSrc); + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION DEBUGLOG(6, "compare block checksum"); @@ -1837,7 +1842,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, dict, (int)dictSize); RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); - if (dctx->frameInfo.contentChecksumFlag) + if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum)) XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= (size_t)decodedSize; @@ -1880,7 +1885,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, dict, (int)dictSize); RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); - if (dctx->frameInfo.contentChecksumFlag) + if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum) XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize); if (dctx->frameInfo.contentSize) dctx->frameRemainingSize -= (size_t)decodedSize; @@ -1945,7 +1950,8 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } /* if (dctx->dStage == dstage_storeSuffix) */ /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ - { U32 const readCRC = LZ4F_readLE32(selectedIn); + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid); @@ -1953,11 +1959,11 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, (void)readCRC; (void)resultCRC; #endif - nextSrcSizeHint = 0; - LZ4F_resetDecompressionContext(dctx); - doAnotherStage = 0; - break; } + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; case dstage_getSFrameSize: if ((srcEnd - srcPtr) >= 4) { diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 1c2d0dd..1bdf6c4 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -355,8 +355,12 @@ typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ typedef struct { - unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. */ - unsigned reserved[3]; /* must be set to zero for forward compatibility */ + unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations. + * This optimization skips storage operations in tmp buffers. */ + unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time. + * Setting this option to 1 once disables all checksums for the rest of the frame. */ + unsigned reserved1; /* must be set to zero for forward compatibility */ + unsigned reserved0; /* idem */ } LZ4F_decompressOptions_t; diff --git a/programs/bench.c b/programs/bench.c index 7836717..5a56e6f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -123,6 +123,7 @@ static size_t g_blockSize = 0; int g_additionalParam = 0; int g_benchSeparately = 0; int g_decodeOnly = 0; +unsigned g_skipChecksums = 0; void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } @@ -140,6 +141,8 @@ void BMK_setBenchSeparately(int separate) { g_benchSeparately = (separate!=0); } void BMK_setDecodeOnlyMode(int set) { g_decodeOnly = (set!=0); } +void BMK_skipChecksums(int skip) { g_skipChecksums = (skip!=0); } + /* ************************************* * Compression state management @@ -318,10 +321,13 @@ LZ4F_decompress_binding(const char* src, char* dst, { size_t dstSize = (size_t)dstCapacity; size_t readSize = (size_t)srcSize; - size_t const decStatus = LZ4F_decompress(g_dctx, + LZ4F_decompressOptions_t dOpt = { 1, 0, 0, 0 }; + size_t decStatus; + dOpt.skipChecksums = g_skipChecksums; + decStatus = LZ4F_decompress(g_dctx, dst, &dstSize, src, &readSize, - NULL /* dOptPtr */); + &dOpt); if ( (decStatus == 0) /* decompression successful */ && ((int)readSize==srcSize) /* consume all input */ ) return (int)dstSize; @@ -775,7 +781,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX; if (g_decodeOnly) { - DISPLAYLEVEL(2, "Benchmark Decompression (only) of LZ4 Frame \n"); + DISPLAYLEVEL(2, "Benchmark Decompression of LZ4 Frame "); + if (g_skipChecksums) { + DISPLAYLEVEL(2, "_without_ checksum even when present \n"); + } else { + DISPLAYLEVEL(2, "+ Checksum when present \n"); + } cLevelLast = cLevel; } if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX; diff --git a/programs/bench.h b/programs/bench.h index 9b0f667..1d81a99 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -47,6 +47,7 @@ void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */ void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */ void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */ +void BMK_skipChecksums(int skip); /* v1.9.4+: only useful for DecodeOnlyMode; do not calculate checksum when present, to save CPU time */ void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */ diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 56c0053..39dc925 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -188,6 +188,9 @@ only the latest one will be applied. * `--[no-]frame-crc`: Select frame checksum (default:enabled) +* `--no-crc`: + Disable both frame and block checksums + * `--[no-]content-size`: Header includes original size (default:not present)<br/> Note : this option can only be activated when the original size can be diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 42132b9..51969fd 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -389,8 +389,9 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; } if ((!strcmp(argument, "--stdout")) || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; } - if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; } - if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; } + if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); BMK_skipChecksums(0); continue; } + if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; } + if (!strcmp(argument, "--no-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); LZ4IO_setBlockChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; } if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; } if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; } if (!strcmp(argument, "--list")) { mode = om_list; continue; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 5644775..8b70b91 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -1072,13 +1072,22 @@ LZ4IO_decompressLZ4F(dRess_t ress, unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; unsigned storedSkips = 0; + LZ4F_decompressOptions_t const dOpt_skipCrc = { 0, 1, 0, 0 }; + const LZ4F_decompressOptions_t* const dOptPtr = + ((prefs->blockChecksum==0) && (prefs->streamChecksum==0)) ? + &dOpt_skipCrc : NULL; /* Init feed with magic number (already consumed from FILE* sFile) */ { size_t inSize = MAGICNUMBER_SIZE; size_t outSize= 0; LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, + ress.dstBuffer, &outSize, + ress.srcBuffer, &inSize, + ress.dictBuffer, ress.dictBufferSize, + dOptPtr); /* set it once, it's enough */ + if (LZ4F_isError(nextToLoad)) + END_PROCESS(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } /* Main Loop */ @@ -1096,8 +1105,13 @@ LZ4IO_decompressLZ4F(dRess_t ress, /* Decode Input (at least partially) */ size_t remaining = readSize - pos; decodedBytes = ress.dstBufferSize; - nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL); - if (LZ4F_isError(nextToLoad)) END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); + nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, + ress.dstBuffer, &decodedBytes, + (char*)(ress.srcBuffer)+pos, &remaining, + ress.dictBuffer, ress.dictBufferSize, + NULL); + if (LZ4F_isError(nextToLoad)) + END_PROCESS(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; /* Write Block */ @@ -1327,6 +1341,10 @@ LZ4IO_decompressDstFile(dRess_t ress, } +/* Note : LZ4IO_decompressFilename() + * can provide total decompression time for the specified fileName. + * This information is not available with LZ4IO_decompressMultipleFilenames(). + */ int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs) { dRess_t const ress = LZ4IO_createDResources(prefs); @@ -1357,6 +1375,9 @@ int LZ4IO_decompressMultipleFilenames( dRess_t ress = LZ4IO_createDResources(prefs); if (outFileName==NULL) END_PROCESS(70, "Memory allocation error"); + if (prefs->blockChecksum==0 && prefs->streamChecksum==0) { + DISPLAYLEVEL(4, "disabling checksum validation during decoding \n"); + } ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs); for (i=0; i<ifntSize; i++) { diff --git a/tests/Makefile b/tests/Makefile index d804f25..4b6ea48 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -376,10 +376,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat $(LZ4) --no-frame-crc < $(FPREFIX)-dg20k | $(LZ4) -d > $(FPREFIX)-dec $(DIFF) -q $(FPREFIX)-dg20k $(FPREFIX)-dec $(DATAGEN) | $(LZ4) -BI | $(LZ4) -t + $(DATAGEN) | $(LZ4) --no-crc | $(LZ4) -t $(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t $(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt $(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t - $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t + $(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t --no-crc @echo "hello world" > $(FPREFIX)-hw $(LZ4) --rm -f $(FPREFIX)-hw $(FPREFIX)-hw.lz4 test ! -f $(FPREFIX)-hw # must fail (--rm) @@ -479,6 +480,7 @@ test-lz4-testmode: lz4 datagen $(DATAGEN) > $(FPREFIX) $(LZ4) -f $(FPREFIX) -c > $(FPREFIX).lz4 $(LZ4) -bdi0 $(FPREFIX).lz4 # test benchmark decode-only mode + $(LZ4) -bdi0 --no-crc $(FPREFIX).lz4 # test benchmark decode-only mode @echo "\n ---- test mode ----" ! $(DATAGEN) | $(LZ4) -t ! $(DATAGEN) | $(LZ4) -tf diff --git a/tests/frametest.c b/tests/frametest.c index ed4186a..3301955 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -906,6 +906,7 @@ size_t test_lz4f_decompression_wBuffers( memset(&dOptions, 0, sizeof(dOptions)); dOptions.stableDst = FUZ_rand(randState) & 1; if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */ + dOptions.skipChecksums = FUZ_rand(randState) & 127; if (sentinelTest) op[oSizeMax] = mark; DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize); |