diff options
-rw-r--r-- | contrib/meson/meson/examples/meson.build | 2 | ||||
-rw-r--r-- | contrib/meson/meson/lib/meson.build | 2 | ||||
-rw-r--r-- | doc/lz4frame_manual.html | 20 | ||||
-rw-r--r-- | examples/frameCompress.c | 134 | ||||
-rw-r--r-- | lib/lz4frame.c | 121 | ||||
-rw-r--r-- | lib/lz4frame.h | 20 | ||||
-rw-r--r-- | ossfuzz/Makefile | 1 | ||||
-rw-r--r-- | ossfuzz/round_trip_frame_uncompressed_fuzzer.c | 138 |
8 files changed, 394 insertions, 44 deletions
diff --git a/contrib/meson/meson/examples/meson.build b/contrib/meson/meson/examples/meson.build index 791bd94..65f54ca 100644 --- a/contrib/meson/meson/examples/meson.build +++ b/contrib/meson/meson/examples/meson.build @@ -26,7 +26,7 @@ foreach e, src : examples executable( e, lz4_source_root / 'examples' / src, - dependencies: liblz4_dep, + dependencies: [liblz4_internal_dep], install: false ) endforeach diff --git a/contrib/meson/meson/lib/meson.build b/contrib/meson/meson/lib/meson.build index 2ff294d..469cd09 100644 --- a/contrib/meson/meson/lib/meson.build +++ b/contrib/meson/meson/lib/meson.build @@ -43,7 +43,7 @@ liblz4_dep = declare_dependency( include_directories: include_directories(lz4_source_root / 'lib') ) -if get_option('tests') or get_option('programs') +if get_option('tests') or get_option('programs') or get_option('examples') liblz4_internal = static_library( 'lz4-internal', objects: liblz4.extract_all_objects(recursive: true), diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 0040c98..0ae5150 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -184,6 +184,8 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); This value is provided by LZ4F_compressBound(). If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). After an error, the state is left in a UB state, and must be re-initialized or freed. + If previously an uncompressed block was written, buffered data is flushed + before appending compressed data is continued. `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). or an error code if it fails (which can be tested using LZ4F_isError()) @@ -342,6 +344,24 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); <pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; </b></pre><BR> +<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); +</b><p> LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary. + Important rule: dstCapacity MUST be large enough to store the entire source buffer as + no compression is done for this operation + If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + After an error, the state is left in a UB state, and must be re-initialized or freed. + If previously a compressed block was written, buffered data is flushed + before appending uncompressed data is continued. + This is only supported when LZ4F_blockIndependent is used + `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + or an error code if it fails (which can be tested using LZ4F_isError()) + +</p></pre><BR> + <a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre> <pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); diff --git a/examples/frameCompress.c b/examples/frameCompress.c index 9eaa4da..25ff729 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -11,8 +11,9 @@ #include <errno.h> #include <assert.h> +#include <getopt.h> #include <lz4frame.h> - +#include <lz4frame_static.h> #define IN_CHUNK_SIZE (16*1024) @@ -57,10 +58,11 @@ static compressResult_t compress_file_internal(FILE* f_in, FILE* f_out, LZ4F_compressionContext_t ctx, void* inBuff, size_t inChunkSize, - void* outBuff, size_t outCapacity) + void* outBuff, size_t outCapacity, + FILE* f_unc, long uncOffset) { compressResult_t result = { 1, 0, 0 }; /* result for an error */ - unsigned long long count_in = 0, count_out; + long long count_in = 0, count_out, bytesToOffset = -1; assert(f_in != NULL); assert(f_out != NULL); assert(ctx != NULL); @@ -81,22 +83,48 @@ compress_file_internal(FILE* f_in, FILE* f_out, /* stream file */ for (;;) { - size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in); + size_t compressedSize; + long long inSize = IN_CHUNK_SIZE; + if (uncOffset >= 0) { + bytesToOffset = uncOffset - count_in; + + /* read only remaining bytes to offset position */ + if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) { + inSize = bytesToOffset; + } + } + + /* input data is at uncompressed data offset */ + if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) { + size_t const readSize = fread(inBuff, 1, inSize, f_unc); + if (readSize == 0) { + uncOffset = -1; + continue; + } + count_in += readSize; + compressedSize = LZ4F_uncompressedUpdate(ctx, + outBuff, outCapacity, + inBuff, readSize, + NULL); + } else { + size_t const readSize = fread(inBuff, 1, inSize, f_in); if (readSize == 0) break; /* nothing left to read from input file */ count_in += readSize; - - size_t const compressedSize = LZ4F_compressUpdate(ctx, + compressedSize = LZ4F_compressUpdate(ctx, outBuff, outCapacity, inBuff, readSize, NULL); - if (LZ4F_isError(compressedSize)) { - printf("Compression failed: error %u \n", (unsigned)compressedSize); - return result; - } - printf("Writing %u bytes\n", (unsigned)compressedSize); - safe_fwrite(outBuff, 1, compressedSize, f_out); - count_out += compressedSize; + } + + if (LZ4F_isError(compressedSize)) { + printf("Compression failed: error %u \n", (unsigned)compressedSize); + return result; + } + + printf("Writing %u bytes\n", (unsigned)compressedSize); + safe_fwrite(outBuff, 1, compressedSize, f_out); + count_out += compressedSize; } /* flush whatever remains within internal buffers */ @@ -120,7 +148,8 @@ compress_file_internal(FILE* f_in, FILE* f_out, } static compressResult_t -compress_file(FILE* f_in, FILE* f_out) +compress_file(FILE* f_in, FILE* f_out, + FILE* f_unc, int uncOffset) { assert(f_in != NULL); assert(f_out != NULL); @@ -137,7 +166,8 @@ compress_file(FILE* f_in, FILE* f_out) result = compress_file_internal(f_in, f_out, ctx, src, IN_CHUNK_SIZE, - outbuff, outbufCapacity); + outbuff, outbufCapacity, + f_unc, uncOffset); } else { printf("error : resource allocation failed \n"); } @@ -305,52 +335,106 @@ static int decompress_file(FILE* f_in, FILE* f_out) } -int compareFiles(FILE* fp0, FILE* fp1) +int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset) { int result = 0; + long bytesRead = 0; + long bytesToOffset = -1; + long b1Size = 1024; while (result==0) { + char b1[b1Size]; + size_t r1; + size_t bytesToRead = sizeof b1; + if (uncOffset >= 0) { + bytesToOffset = uncOffset - bytesRead; + + /* read remainder to offset */ + if (bytesToOffset < b1Size) { + bytesToRead = bytesToOffset; + } + } + char b0[1024]; - char b1[1024]; - size_t const r0 = fread(b0, 1, sizeof(b0), fp0); - size_t const r1 = fread(b1, 1, sizeof(b1), fp1); + size_t r0; + if (bytesToOffset <= 0 && fpUnc) { + bytesToRead = sizeof b1; + r0 = fread(b0, 1,bytesToRead, fpUnc); + } else { + r0 = fread(b0, 1, bytesToRead, fp0); + } + + r1 = fread(b1, 1, r0, fp1); result = (r0 != r1); if (!r0 || !r1) break; if (!result) result = memcmp(b0, b1, r0); + + bytesRead += r1; } return result; } -int main(int argc, const char **argv) { +int main(int argc, char **argv) { char inpFilename[256] = { 0 }; char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; + int uncOffset = -1; + char uncFilename[256] = { 0 }; + int opt; + if (argc < 2) { printf("Please specify input filename\n"); - return 0; + return EXIT_FAILURE; } snprintf(inpFilename, 256, "%s", argv[1]); snprintf(lz4Filename, 256, "%s.lz4", argv[1]); snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + while ((opt = getopt(argc, argv, "o:d:")) != -1) { + switch (opt) { + case 'd': + snprintf(uncFilename, 256, "%s", optarg); + break; + case 'o': + uncOffset = atoi(optarg); + break; + default: + printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]); + printf("-o uncompressed data offset\n"); + printf(" inject uncompressed data at this offset into the lz4 file\n"); + printf("-d uncompressed file\n"); + printf(" file to inject without compression into the lz4 file\n"); + return EXIT_FAILURE; + } + } + printf("inp = [%s]\n", inpFilename); printf("lz4 = [%s]\n", lz4Filename); printf("dec = [%s]\n", decFilename); + if (uncOffset > 0) { + printf("unc = [%s]\n", uncFilename); + printf("ofs = [%i]\n", uncOffset); + } /* compress */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const outFp = fopen(lz4Filename, "wb"); + FILE* const uncFp = fopen(uncFilename, "rb"); printf("compress : %s -> %s\n", inpFilename, lz4Filename); - compressResult_t const ret = compress_file(inpFp, outFp); + compressResult_t const ret = compress_file( + inpFp, outFp, + uncFp, uncOffset); fclose(outFp); fclose(inpFp); + if (uncFp) + fclose(uncFp); if (ret.error) { printf("compress : failed with code %i\n", ret.error); @@ -383,12 +467,16 @@ int main(int argc, const char **argv) { /* verify */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const decFp = fopen(decFilename, "rb"); + FILE* const uncFp = fopen(uncFilename, "rb"); printf("verify : %s <-> %s\n", inpFilename, decFilename); - int const cmp = compareFiles(inpFp, decFp); + int const cmp = compareFiles(inpFp, decFp, + uncFp, uncOffset); fclose(decFp); fclose(inpFp); + if (uncFp) + fclose(uncFp); if (cmp) { printf("corruption detected : decompressed file differs from original\n"); diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a0275ca..80e244b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -220,6 +220,9 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checks /*-************************************ * Structures and local types **************************************/ + +typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t; + typedef struct LZ4F_cctx_s { LZ4F_preferences_t prefs; @@ -236,6 +239,7 @@ typedef struct LZ4F_cctx_s void* lz4CtxPtr; U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + LZ4F_blockCompression_t blockCompression; } LZ4F_cctx_t; @@ -757,14 +761,27 @@ static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level, const LZ4F_CDict* cdict, - LZ4F_blockChecksum_t crcFlag) + LZ4F_blockChecksum_t crcFlag, + LZ4F_blockCompression_t blockCompression) { BYTE* const cSizePtr = (BYTE*)dst; - U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), - (int)(srcSize), (int)(srcSize-1), - level, cdict); - if (cSize == 0) { /* compression failed */ - DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); + U32 cSize; + if (compress != NULL) { + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), + (int)(srcSize), (int)(srcSize-1), + level, cdict); + } else { + cSize = (U32)srcSize; + /* force no compression if compress callback is null */ + blockCompression = LZ4B_UNCOMPRESSED; + } + + if (cSize == 0) { /* compression failed */ + DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize); + blockCompression = LZ4B_UNCOMPRESSED; + } + + if (blockCompression == LZ4B_UNCOMPRESSED) { cSize = (U32)srcSize; LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); memcpy(cSizePtr+BHSize, src, srcSize); @@ -836,21 +853,25 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; -/*! LZ4F_compressUpdate() : + + /*! LZ4F_compressUpdateImpl() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. * When successful, the function always entirely consumes @srcBuffer. * src data is either buffered or compressed into @dstBuffer. - * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * If the block compression does not match the compression of the previous block, the old data is flushed + * and operations continue with the new compression mode. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on. * @compressOptionsPtr is optional : provide NULL to mean "default". * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. * or an error code if it fails (which can be tested using LZ4F_isError()) * After an error, the state is left in a UB state, and must be re-initialized. */ -size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, - void* dstBuffer, size_t dstCapacity, +static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* compressOptionsPtr) -{ + const LZ4F_compressOptions_t* compressOptionsPtr, + LZ4F_blockCompression_t blockCompression) + { size_t const blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; @@ -858,12 +879,23 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, BYTE* dstPtr = dstStart; LZ4F_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - + size_t bytesWritten; DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */ if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize) + RETURN_ERROR(dstMaxSize_tooSmall); + + /* flush currently written block, to continue with new block compression */ + if (cctxPtr->blockCompression != blockCompression) { + bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + dstPtr += bytesWritten; + cctxPtr->blockCompression = blockCompression; + } + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; /* complete tmp buffer */ @@ -886,8 +918,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); - + cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression); if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; } } @@ -899,7 +930,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->prefs.frameInfo.blockChecksumFlag, + blockCompression); srcPtr += blockSize; } @@ -910,12 +942,15 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, srcPtr, (size_t)(srcEnd - srcPtr), compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); - srcPtr = srcEnd; + cctxPtr->prefs.frameInfo.blockChecksumFlag, + blockCompression); + srcPtr = srcEnd; } /* preserve dictionary within @tmpBuff whenever necessary */ if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */ + assert(blockCompression == LZ4B_COMPRESSED); if (compressOptionsPtr->stableSrc) { cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ } else { @@ -951,6 +986,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, return (size_t)(dstPtr - dstStart); } +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_COMPRESSED); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * This is only supported when LZ4F_blockIndependent is used + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) { + RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid); + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_UNCOMPRESSED); +} + /*! LZ4F_flush() : * When compressed data must be sent immediately, without waiting for a block to be filled, @@ -981,7 +1063,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, cctxPtr->cdict, - cctxPtr->prefs.frameInfo.blockChecksumFlag); + cctxPtr->prefs.frameInfo.blockChecksumFlag, + cctxPtr->blockCompression); assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity)); if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 6bf20e4..2c5a559 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -306,6 +306,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) @@ -545,6 +547,24 @@ LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); +/*! LZ4F_uncompressedUpdate() : + * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary. + * Important rule: dstCapacity MUST be large enough to store the entire source buffer as + * no compression is done for this operation + * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously a compressed block was written, buffered data is flushed + * before appending uncompressed data is continued. + * This is only supported when LZ4F_blockIndependent is used + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); + /********************************** * Bulk processing dictionary API *********************************/ diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile index 2ec1675..deb2938 100644 --- a/ossfuzz/Makefile +++ b/ossfuzz/Makefile @@ -45,6 +45,7 @@ FUZZERS := \ round_trip_hc_fuzzer \ compress_frame_fuzzer \ round_trip_frame_fuzzer \ + round_trip_frame_uncompressed_fuzzer \ decompress_frame_fuzzer .PHONY: all diff --git a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c new file mode 100644 index 0000000..e578052 --- /dev/null +++ b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c @@ -0,0 +1,138 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_data_producer.h" +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4_helpers.h" +#include "lz4frame.h" +#include "lz4frame_static.h" + +static void decompress(LZ4F_dctx *dctx, void *src, void *dst, + size_t dstCapacity, size_t readSize) { + size_t ret = 1; + const void *srcPtr = (const char *)src; + void *dstPtr = (char *)dst; + const void *const srcEnd = (const char *)srcPtr + readSize; + + while (ret != 0) { + while (srcPtr < srcEnd && ret != 0) { + /* Any data within dst has been flushed at this stage */ + size_t dstSize = dstCapacity; + size_t srcSize = (const char *)srcEnd - (const char *)srcPtr; + ret = LZ4F_decompress(dctx, dstPtr, &dstSize, srcPtr, &srcSize, + /* LZ4F_decompressOptions_t */ NULL); + FUZZ_ASSERT(!LZ4F_isError(ret)); + + /* Update input */ + srcPtr = (const char *)srcPtr + srcSize; + dstPtr = (char *)dstPtr + dstSize; + } + + FUZZ_ASSERT(srcPtr <= srcEnd); + } +} + +static void compress_round_trip(const uint8_t* data, size_t size, + FUZZ_dataProducer_t *producer, LZ4F_preferences_t const prefs) { + size = FUZZ_dataProducer_remainingBytes(producer); + + uint8_t *uncompressedData = malloc(size); + size_t uncompressedOffset = rand() % (size + 1); + + FUZZ_dataProducer_t *uncompressedProducer = + FUZZ_dataProducer_create(uncompressedData, size); + size_t uncompressedSize = + FUZZ_dataProducer_remainingBytes(uncompressedProducer); + + size_t const dstCapacity = + LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs) + + uncompressedSize; + char *const dst = (char *)malloc(dstCapacity); + size_t rtCapacity = dstCapacity; + char *const rt = (char *)malloc(rtCapacity); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + LZ4F_compressionContext_t ctx; + size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + FUZZ_ASSERT(!LZ4F_isError(ctxCreation)); + + size_t const headerSize = LZ4F_compressBegin(ctx, dst, dstCapacity, &prefs); + FUZZ_ASSERT(!LZ4F_isError(headerSize)); + size_t compressedSize = headerSize; + + /* Compress data before uncompressed offset */ + size_t lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity, + data, uncompressedOffset, NULL); + FUZZ_ASSERT(!LZ4F_isError(lz4Return)); + compressedSize += lz4Return; + + /* Add uncompressed data */ + lz4Return = LZ4F_uncompressedUpdate(ctx, dst + compressedSize, dstCapacity, + uncompressedData, uncompressedSize, NULL); + FUZZ_ASSERT(!LZ4F_isError(lz4Return)); + compressedSize += lz4Return; + + /* Compress data after uncompressed offset */ + lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity, + data + uncompressedOffset, + size - uncompressedOffset, NULL); + FUZZ_ASSERT(!LZ4F_isError(lz4Return)); + compressedSize += lz4Return; + + /* Finish compression */ + lz4Return = LZ4F_compressEnd(ctx, dst + compressedSize, dstCapacity, NULL); + FUZZ_ASSERT(!LZ4F_isError(lz4Return)); + compressedSize += lz4Return; + + LZ4F_decompressOptions_t opts; + memset(&opts, 0, sizeof(opts)); + opts.stableDst = 1; + LZ4F_dctx *dctx; + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + FUZZ_ASSERT(dctx); + + decompress(dctx, dst, rt, rtCapacity, compressedSize); + + LZ4F_freeDecompressionContext(dctx); + + char *const expectedData = (char *)malloc(size + uncompressedSize); + memcpy(expectedData, data, uncompressedOffset); + memcpy(expectedData + uncompressedOffset, uncompressedData, uncompressedSize); + memcpy(expectedData + uncompressedOffset + uncompressedSize, + data + uncompressedOffset, size - uncompressedOffset); + + FUZZ_ASSERT_MSG(!memcmp(expectedData, rt, size), "Corruption!"); + free(expectedData); + + free(dst); + free(rt); + free(uncompressedData); + + FUZZ_dataProducer_free(producer); + FUZZ_dataProducer_free(uncompressedProducer); + LZ4F_freeCompressionContext(ctx); +} + +static void compress_independent_block_mode(const uint8_t* data, size_t size) { + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); + LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer); + prefs.frameInfo.blockMode = LZ4F_blockIndependent; + compress_round_trip(data, size, producer, prefs); +} + + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + compress_independent_block_mode(data, size); + return 0; +} |