From 10d2e1c69465ac51d210909ca7a447a1397736a1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 17 Sep 2020 14:43:02 -0700 Subject: fixed lz4frame with blocks of size 1 properly track history --- lib/lz4.c | 40 +++++++++++++++++++--------------------- lib/lz4frame.c | 5 ++++- tests/frametest.c | 6 ++++-- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 6d918ea..2222d53 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -823,7 +823,6 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, * Presumed already validated at this stage: * - source != NULL * - inputSize > 0 - * - outputDirective == notLimited || maxOutputSize > 0 */ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( LZ4_stream_t_internal* const cctx, @@ -876,6 +875,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( assert(ip != NULL); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -1216,44 +1216,41 @@ _last_literals: /** LZ4_compress_generic() : * inlined, to ensure branches are decided at compilation time; - * takes care of input == (NULL, 0) and outputSize == 0 + * takes care of src == (NULL, 0) * and forward the rest to LZ4_compress_generic_validated */ LZ4_FORCE_INLINE int LZ4_compress_generic( LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, + const char* const src, + char* const dst, + const int srcSize, int *inputConsumed, /* only written when outputDirective == fillOutput */ - const int maxOutputSize, + const int dstCapacity, const limitedOutput_directive outputDirective, const tableType_t tableType, const dict_directive dictDirective, const dictIssue_directive dictIssue, const int acceleration) { - DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, maxOutputSize=%i", - inputSize, maxOutputSize); - - /* If init conditions are not met, we don't have to mark stream - * as having dirty context, since no action was taken yet */ - if (outputDirective != notLimited && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ - assert(dest != NULL); + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ - if (inputSize == 0) { /* source == NULL supported if inputSize == 0 */ - assert(outputDirective == notLimited || maxOutputSize >= 1); - dest[0] = 0; + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; if (outputDirective == fillOutput) { assert (inputConsumed != NULL); *inputConsumed = 0; } return 1; } - assert(source != NULL); + assert(src != NULL); - return LZ4_compress_generic_validated(cctx, source, dest, inputSize, - inputConsumed, /* only written when outputDirective == fillOutput */ - maxOutputSize, outputDirective, + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, tableType, dictDirective, dictIssue, acceleration); } @@ -2124,6 +2121,7 @@ LZ4_decompress_generic( /* end of decoding */ if (endOnInput) { + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ diff --git a/lib/lz4frame.c b/lib/lz4frame.c index e11f1c8..890fc96 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -750,6 +750,7 @@ static size_t LZ4F_makeBlock(void* dst, (int)(srcSize), (int)(srcSize-1), level, cdict); if (cSize == 0) { /* compression failed */ + DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+BHSize, src, srcSize); @@ -1487,15 +1488,17 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, size_t const nextCBlockSize = blockHeader & 0x7FFFFFFFU; size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; if (blockHeader==0) { /* frameEnd signal, no more block */ + DEBUGLOG(5, "end of frame"); dctx->dStage = dstage_getSuffix; break; } if (nextCBlockSize > dctx->maxBlockSize) { return err0r(LZ4F_ERROR_maxBlockSize_invalid); } - if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { + if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) { /* next block is uncompressed */ dctx->tmpInTarget = nextCBlockSize; + DEBUGLOG(5, "next block is uncompressed (size %u)", (U32)nextCBlockSize); if (dctx->frameInfo.blockChecksumFlag) { (void)XXH32_reset(&dctx->blockChecksum, 0); } diff --git a/tests/frametest.c b/tests/frametest.c index 236a98c..54c7fd2 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -785,7 +785,7 @@ typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e; static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario) { - if (displayLevel >= 5) { + if (displayLevel >= 2) { size_t p=0; const BYTE* b1=(const BYTE*)buff1; const BYTE* b2=(const BYTE*)buff2; @@ -862,8 +862,10 @@ size_t test_lz4f_decompression_wBuffers( "Decompression overwrites beyond assigned dst size", op[oSizeMax], mark); } - if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) + if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) { + DISPLAYLEVEL(2, "checksum error detected \n"); locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); + } if (LZ4F_isError(moreToFlush)) return moreToFlush; XXH64_update(&xxh64, op, oSize); -- cgit v0.12