diff options
author | Alexander Mohr <alexander.m.mohr@mercedes-benz.com> | 2022-06-03 13:38:54 (GMT) |
---|---|---|
committer | Alexander Mohr <alexander.m.mohr@mercedes-benz.com> | 2022-06-09 15:08:01 (GMT) |
commit | 4aeb5020c35e2464c25eb69d8bf6c7645b8faf20 (patch) | |
tree | d23ce84bf30f996bf6c7842445078986be63c628 | |
parent | 14d3b6342f1dca15fe36f952434ecf3c9b8329a0 (diff) | |
download | lz4-4aeb5020c35e2464c25eb69d8bf6c7645b8faf20.zip lz4-4aeb5020c35e2464c25eb69d8bf6c7645b8faf20.tar.gz lz4-4aeb5020c35e2464c25eb69d8bf6c7645b8faf20.tar.bz2 |
frame-api: add method to insert uncomressed data
new method `uncompressed_update` allows to insert blocks without
compression into the lz4 stream.
The usage is documented in the frameCompress example
Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
-rw-r--r-- | doc/lz4frame_manual.html | 5 | ||||
-rw-r--r-- | examples/frameCompress.c | 133 | ||||
-rw-r--r-- | lib/lz4.c | 14 | ||||
-rw-r--r-- | lib/lz4.h | 2 | ||||
-rw-r--r-- | lib/lz4frame.c | 228 | ||||
-rw-r--r-- | lib/lz4frame.h | 23 | ||||
-rw-r--r-- | lib/lz4hc.c | 17 | ||||
-rw-r--r-- | lib/lz4hc.h | 2 |
8 files changed, 329 insertions, 95 deletions
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 0040c98..9490b51 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -75,6 +75,11 @@ LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t; </b></pre><BR> +<pre><b>typedef enum { + LZ4B_COMPRESSED, + LZ4B_UNCOMPRESSED, +} LZ4F_blockCompression_t; +</b></pre><BR> <pre><b>typedef struct { LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b> LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b> diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 9eaa4da..3219014 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -11,9 +11,9 @@ #include <errno.h> #include <assert.h> +#include <getopt.h> #include <lz4frame.h> - #define IN_CHUNK_SIZE (16*1024) static const LZ4F_preferences_t kPrefs = { @@ -57,10 +57,11 @@ static compressResult_t compress_file_internal(FILE* f_in, FILE* f_out, LZ4F_compressionContext_t ctx, void* inBuff, size_t inChunkSize, - void* outBuff, size_t outCapacity) + void* outBuff, size_t outCapacity, + FILE* f_unc, long uncOffset) { compressResult_t result = { 1, 0, 0 }; /* result for an error */ - unsigned long long count_in = 0, count_out; + long long count_in = 0, count_out, bytesToOffset = -1; assert(f_in != NULL); assert(f_out != NULL); assert(ctx != NULL); @@ -81,22 +82,48 @@ compress_file_internal(FILE* f_in, FILE* f_out, /* stream file */ for (;;) { - size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in); + size_t compressedSize; + long long inSize = IN_CHUNK_SIZE; + if (uncOffset >= 0) { + bytesToOffset = uncOffset - count_in; + + /* read only remaining bytes to offset position */ + if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) { + inSize = bytesToOffset; + } + } + + /* input data is at uncompressed data offset */ + if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) { + size_t const readSize = fread(inBuff, 1, inSize, f_unc); + if (readSize == 0) { + uncOffset = -1; + continue; + } + count_in += readSize; + compressedSize = LZ4F_uncompressedUpdate(ctx, + outBuff, outCapacity, + inBuff, readSize, + NULL); + } else { + size_t const readSize = fread(inBuff, 1, inSize, f_in); if (readSize == 0) break; /* nothing left to read from input file */ count_in += readSize; - - size_t const compressedSize = LZ4F_compressUpdate(ctx, + compressedSize = LZ4F_compressUpdate(ctx, outBuff, outCapacity, inBuff, readSize, NULL); - if (LZ4F_isError(compressedSize)) { - printf("Compression failed: error %u \n", (unsigned)compressedSize); - return result; - } - printf("Writing %u bytes\n", (unsigned)compressedSize); - safe_fwrite(outBuff, 1, compressedSize, f_out); - count_out += compressedSize; + } + + if (LZ4F_isError(compressedSize)) { + printf("Compression failed: error %u \n", (unsigned)compressedSize); + return result; + } + + printf("Writing %u bytes\n", (unsigned)compressedSize); + safe_fwrite(outBuff, 1, compressedSize, f_out); + count_out += compressedSize; } /* flush whatever remains within internal buffers */ @@ -120,7 +147,8 @@ compress_file_internal(FILE* f_in, FILE* f_out, } static compressResult_t -compress_file(FILE* f_in, FILE* f_out) +compress_file(FILE* f_in, FILE* f_out, + FILE* f_unc, int uncOffset) { assert(f_in != NULL); assert(f_out != NULL); @@ -137,7 +165,8 @@ compress_file(FILE* f_in, FILE* f_out) result = compress_file_internal(f_in, f_out, ctx, src, IN_CHUNK_SIZE, - outbuff, outbufCapacity); + outbuff, outbufCapacity, + f_unc, uncOffset); } else { printf("error : resource allocation failed \n"); } @@ -305,52 +334,106 @@ static int decompress_file(FILE* f_in, FILE* f_out) } -int compareFiles(FILE* fp0, FILE* fp1) +int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset) { int result = 0; + long bytesRead = 0; + long bytesToOffset = -1; + long b1Size = 1024; while (result==0) { + char b1[b1Size]; + size_t r1; + size_t bytesToRead = sizeof b1; + if (uncOffset >= 0) { + bytesToOffset = uncOffset - bytesRead; + + /* read remainder to offset */ + if (bytesToOffset < b1Size) { + bytesToRead = bytesToOffset; + } + } + char b0[1024]; - char b1[1024]; - size_t const r0 = fread(b0, 1, sizeof(b0), fp0); - size_t const r1 = fread(b1, 1, sizeof(b1), fp1); + size_t r0; + if (bytesToOffset <= 0 && fpUnc) { + bytesToRead = sizeof b1; + r0 = fread(b0, 1,bytesToRead, fpUnc); + } else { + r0 = fread(b0, 1, bytesToRead, fp0); + } + + r1 = fread(b1, 1, r0, fp1); result = (r0 != r1); if (!r0 || !r1) break; if (!result) result = memcmp(b0, b1, r0); + + bytesRead += r1; } return result; } -int main(int argc, const char **argv) { +int main(int argc, char **argv) { char inpFilename[256] = { 0 }; char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; + int uncOffset = -1; + char uncFilename[256] = { 0 }; + int opt; + if (argc < 2) { printf("Please specify input filename\n"); - return 0; + return EXIT_FAILURE; } snprintf(inpFilename, 256, "%s", argv[1]); snprintf(lz4Filename, 256, "%s.lz4", argv[1]); snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + while ((opt = getopt(argc, argv, "o:d:")) != -1) { + switch (opt) { + case 'd': + snprintf(uncFilename, 256, "%s", optarg); + break; + case 'o': + uncOffset = atoi(optarg); + break; + default: + printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]); + printf("-o uncompressed data offset\n"); + printf(" inject uncompressed data at this offset into the lz4 file\n"); + printf("-d uncompressed file\n"); + printf(" file to inject without compression into the lz4 file\n"); + return EXIT_FAILURE; + } + } + printf("inp = [%s]\n", inpFilename); printf("lz4 = [%s]\n", lz4Filename); printf("dec = [%s]\n", decFilename); + if (uncOffset > 0) { + printf("unc = [%s]\n", uncFilename); + printf("ofs = [%i]\n", uncOffset); + } /* compress */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const outFp = fopen(lz4Filename, "wb"); + FILE* const uncFp = fopen(uncFilename, "rb"); printf("compress : %s -> %s\n", inpFilename, lz4Filename); - compressResult_t const ret = compress_file(inpFp, outFp); + compressResult_t const ret = compress_file( + inpFp, outFp, + uncFp, uncOffset); fclose(outFp); fclose(inpFp); + if (uncFp) + fclose(uncFp); if (ret.error) { printf("compress : failed with code %i\n", ret.error); @@ -383,12 +466,16 @@ int main(int argc, const char **argv) { /* verify */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const decFp = fopen(decFilename, "rb"); + FILE* const uncFp = fopen(uncFilename, "rb"); printf("verify : %s <-> %s\n", inpFilename, decFilename); - int const cmp = compareFiles(inpFp, decFp); + int const cmp = compareFiles(inpFp, decFp, + uncFp, uncOffset); fclose(decFp); fclose(inpFp); + if (uncFp) + fclose(uncFp); if (cmp) { printf("corruption detected : decompressed file differs from original\n"); @@ -1679,6 +1679,15 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* return result; } +int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + return dictSize; +} /*! LZ4_saveDict() : * If previously compressed data block is not guaranteed to remain available at its memory location, @@ -1690,12 +1699,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - + dictSize = LZ4_DictSize(LZ4_dict, dictSize); DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); - if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } - if (safeBuffer == NULL) assert(dictSize == 0); if (dictSize > 0) { const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; @@ -346,6 +346,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); +LZ4LIB_API int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize); + /*! LZ4_saveDict() : * If last 64KB data cannot be guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a0275ca..bcf9629 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -129,7 +129,8 @@ static int g_debuglog_enable = 1; **************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> - typedef uint8_t BYTE; +#include <stdio.h> +typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; @@ -236,6 +237,7 @@ typedef struct LZ4F_cctx_s void* lz4CtxPtr; U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + LZ4F_blockCompression_t blockCompression; } LZ4F_cctx_t; @@ -757,14 +759,27 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict, - LZ4F_blockChecksum_t crcFlag) + LZ4F_blockChecksum_t crcFlag, + LZ4F_blockCompression_t blockCompression) { BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), - (int)(srcSize), (int)(srcSize-1), - level, cdict); - if (cSize == 0) { /* compression failed */ - DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); + U32 cSize; + if (compress != NULL) { + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), + (int)(srcSize), (int)(srcSize-1), + level, cdict); + } else { + cSize = (U32)srcSize; + /* force no compression if compress callback is null */ + blockCompression = LZ4B_UNCOMPRESSED; + } + + if (cSize == 0) { /* compression failed */ + DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); + blockCompression = LZ4B_UNCOMPRESSED; + } + + if (blockCompression == LZ4B_UNCOMPRESSED) { cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+BHSize, src, srcSize); @@ -824,33 +839,48 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev return LZ4F_compressBlockHC_continue; } +static int LZ4F_maxDictSize(void) { + return 64 KB; +} + /* Save history (up to 64KB) into @tmpBuff */ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) - return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); - return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize()); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize()); +} + +static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + return LZ4_DictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize()); + return LZ4_DictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize()); } typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; -/*! LZ4F_compressUpdate() : + + /*! LZ4F_compressUpdateImpl() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. * When successful, the function always entirely consumes @srcBuffer. * src data is either buffered or compressed into @dstBuffer. - * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * If the block compression does not match the compression of the previous block, the old data is flushed + * and operations continue with the new compression mode. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on. * @compressOptionsPtr is optional : provide NULL to mean "default". * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. * or an error code if it fails (which can be tested using LZ4F_isError()) * After an error, the state is left in a UB state, and must be re-initialized. */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, +static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* compressOptionsPtr) -{ + const LZ4F_compressOptions_t* compressOptionsPtr, + LZ4F_blockCompression_t blockCompression) + { size_t const blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; @@ -858,49 +888,62 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, BYTE* dstPtr = dstStart; LZ4F_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - + size_t bytesWritten = 0; DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); + /* flush currently written block, to continue with new block compression */ + if (cctxPtr->blockCompression != blockCompression) { + bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + cctxPtr->blockCompression = blockCompression; + } + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */ - if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) - RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_COMPRESSED && + dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) + RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize) + RETURN_ERROR(dstMaxSize_tooSmall); + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ - size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; - assert(blockSize > cctxPtr->tmpInSize); - if (sizeToCopy > srcSize) { - /* add src to tmpIn buffer */ - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); - srcPtr = srcEnd; - cctxPtr->tmpInSize += srcSize; - /* still needs some CRC */ - } else { - /* complete tmpIn block and then compress it */ - lastBlockCompressed = fromTmpBuffer; - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); - srcPtr += sizeToCopy; - - dstPtr += LZ4F_makeBlock(dstPtr, - cctxPtr->tmpIn, blockSize, - compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, - cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); - - if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; - cctxPtr->tmpInSize = 0; - } } + size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; + assert(blockSize > cctxPtr->tmpInSize); + if (sizeToCopy > srcSize) { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } else { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; - while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full blocks */ - lastBlockCompressed = fromSrcBuffer; dstPtr += LZ4F_makeBlock(dstPtr, - srcPtr, blockSize, + cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr += blockSize; + cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression); + + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) { + /* compress full blocks */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression); + srcPtr += blockSize; } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { @@ -910,19 +953,29 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, srcPtr, (size_t)(srcEnd - srcPtr), compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr = srcEnd; + cctxPtr->prefs.frameInfo.blockChecksumFlag, + blockCompression); + srcPtr = srcEnd; } /* preserve dictionary within @tmpBuff whenever necessary */ if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { - if (compressOptionsPtr->stableSrc) { - cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ + if (compressOptionsPtr->stableSrc) { + cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ + } else { + int realDictSize; + if (blockCompression == LZ4B_COMPRESSED) { + realDictSize = LZ4F_localSaveDict(cctxPtr); } else { - int const realDictSize = LZ4F_localSaveDict(cctxPtr); - assert(0 <= realDictSize && realDictSize <= 64 KB); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate + * this is only relevant if linked block mode + * */ + realDictSize = LZ4F_localDictSize(cctxPtr); } + + assert(0 <= realDictSize && realDictSize <= 64 KB); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } } /* keep tmpIn within limits */ @@ -931,24 +984,75 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, { /* only preserve 64KB within internal buffer. Ensures there is enough room for next block. * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */ - int const realDictSize = LZ4F_localSaveDict(cctxPtr); + int realDictSize; + if (blockCompression == LZ4B_COMPRESSED) { + realDictSize = LZ4F_localSaveDict(cctxPtr); + } else { + /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/ + realDictSize = LZ4F_maxDictSize(); + } cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)); } /* some input data left, necessarily < blockSize */ if (srcPtr < srcEnd) { - /* fill tmp buffer */ - size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); - memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); - cctxPtr->tmpInSize = sizeToCopy; + /* fill tmp buffer */ + size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) - (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; - return (size_t)(dstPtr - dstStart); + return bytesWritten + (size_t)(dstPtr - dstStart); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_COMPRESSED); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) { + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_UNCOMPRESSED); } @@ -981,7 +1085,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression); assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity)); if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 74f19cd..18d33e1 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -160,6 +160,11 @@ typedef enum { LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t; +typedef enum { + LZ4B_COMPRESSED, + LZ4B_UNCOMPRESSED +} LZ4F_blockCompression_t; + #ifdef LZ4F_ENABLE_OBSOLETE_ENUMS typedef LZ4F_blockSizeID_t blockSizeID_t; typedef LZ4F_blockMode_t blockMode_t; @@ -303,6 +308,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) @@ -312,6 +319,22 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); +/*! LZ4F_uncompressedUpdate() : + * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary. + * Important rule: dstCapacity MUST be large enough to store the entire source buffer as + * no compression is done for this operation + * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously a compressed block was written, buffered data is flushed + * before appending uncompressed data is continued. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); /*! LZ4F_flush() : * When data must be generated and sent immediately, without waiting for a block to be completely filled, * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 99650a6..da806ef 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1154,6 +1154,16 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); } +int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) { + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; + int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + assert(prefixSize >= 0); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + return dictSize; +} /* LZ4_saveDictHC : @@ -1164,12 +1174,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; - int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); - DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); - assert(prefixSize >= 0); - if (dictSize > 64 KB) dictSize = 64 KB; - if (dictSize < 4) dictSize = 0; - if (dictSize > prefixSize) dictSize = prefixSize; + dictSize = LZ4_DictHCSize(LZ4_streamHCPtr, dictSize); if (safeBuffer == NULL) assert(dictSize == 0); if (dictSize > 0) memmove(safeBuffer, streamPtr->end - dictSize, dictSize); diff --git a/lib/lz4hc.h b/lib/lz4hc.h index f4afc9b..11671dc 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -173,6 +173,8 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr const char* src, char* dst, int* srcSizePtr, int targetDstSize); +LZ4LIB_API int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize); + LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); |