diff options
author | Yann Collet <Cyan4973@users.noreply.github.com> | 2019-07-19 04:53:33 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-07-19 04:53:33 (GMT) |
commit | 316f2b6f4d74ed542af7687460af637a4901e09e (patch) | |
tree | 5e178908dd7d63670366c78176b8621dba9d394c | |
parent | 19b099986aeccc12bb46ad207fe8de5b36bdb7bc (diff) | |
parent | d28159c025829fc70a295b727e32f899a9e0c7c5 (diff) | |
download | lz4-316f2b6f4d74ed542af7687460af637a4901e09e.zip lz4-316f2b6f4d74ed542af7687460af637a4901e09e.tar.gz lz4-316f2b6f4d74ed542af7687460af637a4901e09e.tar.bz2 |
Merge pull request #762 from terrelln/frame-fuzz
[fuzz] Add LZ4 frame fuzzers
-rw-r--r-- | lib/lz4.c | 2 | ||||
-rw-r--r-- | lib/lz4frame.c | 12 | ||||
-rw-r--r-- | lib/lz4hc.c | 3 | ||||
-rw-r--r-- | ossfuzz/Makefile | 18 | ||||
-rw-r--r-- | ossfuzz/compress_frame_fuzzer.c | 42 | ||||
-rw-r--r-- | ossfuzz/compress_hc_fuzzer.c | 57 | ||||
-rw-r--r-- | ossfuzz/decompress_frame_fuzzer.c | 67 | ||||
-rw-r--r-- | ossfuzz/fuzz_helpers.h | 9 | ||||
-rw-r--r-- | ossfuzz/lz4_helpers.c | 51 | ||||
-rw-r--r-- | ossfuzz/lz4_helpers.h | 13 | ||||
-rwxr-xr-x | ossfuzz/ossfuzz.sh | 4 | ||||
-rw-r--r-- | ossfuzz/round_trip_frame_fuzzer.c | 39 | ||||
-rw-r--r-- | ossfuzz/round_trip_hc_fuzzer.c | 39 | ||||
-rw-r--r-- | ossfuzz/round_trip_stream_fuzzer.c | 302 | ||||
-rwxr-xr-x | ossfuzz/travisoss.sh | 5 |
15 files changed, 652 insertions, 11 deletions
@@ -1429,7 +1429,7 @@ void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dict */ LZ4_resetStream_fast(workingStream); - if (dictionaryStream != NULL) { + if (dictionaryStream != NULL && dictionaryStream->internal_donotuse.dictSize > 0) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table * entry can take that indicate a miss. In that case, we need diff --git a/lib/lz4frame.c b/lib/lz4frame.c index cc7f2d5..13c1ae9 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1131,8 +1131,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif dctx->frameInfo.frameType = LZ4F_frame; /* Flags */ @@ -1171,10 +1173,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize /* check header */ assert(frameHeaderSize > 5); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } +#endif /* save */ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; @@ -1211,8 +1215,10 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize) return 8; /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; @@ -1555,8 +1561,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } { U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != calcCRC) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#endif } } dctx->dStage = dstage_getBlockHeader; /* new block */ break; @@ -1595,8 +1603,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readBlockCrc != calcBlockCrc) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#endif } } if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { @@ -1724,8 +1734,10 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ { U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); +#endif nextSrcSizeHint = 0; LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 46c20bc..29288a5 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1005,6 +1005,9 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile index 4d24944..6875eb6 100644 --- a/ossfuzz/Makefile +++ b/ossfuzz/Makefile @@ -33,7 +33,21 @@ DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) LZ4_CFLAGS = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) -LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ +LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + +FUZZERS := \ + compress_fuzzer \ + decompress_fuzzer \ + round_trip_fuzzer \ + round_trip_stream_fuzzer \ + compress_hc_fuzzer \ + round_trip_hc_fuzzer \ + compress_frame_fuzzer \ + round_trip_frame_fuzzer \ + decompress_frame_fuzzer + +all: $(FUZZERS) # Include a rule to build the static library if calling this target # directly. @@ -44,7 +58,7 @@ $(LZ4DIR)/liblz4.a: $(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@ # Generic rule for generating fuzzers -%_fuzzer: %_fuzzer.o $(LZ4DIR)/liblz4.a +%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a # Compile the standalone code just in case. The OSS-Fuzz code might # override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer" $(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o diff --git a/ossfuzz/compress_frame_fuzzer.c b/ossfuzz/compress_frame_fuzzer.c new file mode 100644 index 0000000..75c609f --- /dev/null +++ b/ossfuzz/compress_frame_fuzzer.c @@ -0,0 +1,42 @@ +/** + * This fuzz target attempts to compress the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4frame.h" +#include "lz4_helpers.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed); + size_t const compressBound = LZ4F_compressFrameBound(size, &prefs); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* If compression succeeds it must round trip correctly. */ + size_t const dstSize = + LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); + if (!LZ4F_isError(dstSize)) { + size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/compress_hc_fuzzer.c b/ossfuzz/compress_hc_fuzzer.c new file mode 100644 index 0000000..4841367 --- /dev/null +++ b/ossfuzz/compress_hc_fuzzer.c @@ -0,0 +1,57 @@ +/** + * This fuzz target attempts to compress the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4hc.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size)); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* If compression succeeds it must round trip correctly. */ + { + int const dstSize = LZ4_compress_HC((const char*)data, dst, size, + dstCapacity, level); + if (dstSize > 0) { + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + } + } + + if (dstCapacity > 0) { + /* Compression succeeds and must round trip correctly. */ + void* state = malloc(LZ4_sizeofStateHC()); + FUZZ_ASSERT(state); + int compressedSize = size; + int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data, + dst, &compressedSize, + dstCapacity, level); + FUZZ_ASSERT(dstSize > 0); + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!"); + free(state); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/decompress_frame_fuzzer.c b/ossfuzz/decompress_frame_fuzzer.c new file mode 100644 index 0000000..bda25b0 --- /dev/null +++ b/ossfuzz/decompress_frame_fuzzer.c @@ -0,0 +1,67 @@ +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#include "lz4_helpers.h" + +static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* opts) +{ + LZ4F_resetDecompressionContext(dctx); + if (dictSize == 0) + LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts); + else + LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize, + dict, dictSize, opts); +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size); + size_t const largeDictSize = 64 * 1024; + size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize); + char* const dst = (char*)malloc(dstCapacity); + char* const dict = (char*)malloc(dictSize); + LZ4F_decompressOptions_t opts; + LZ4F_dctx* dctx; + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + + FUZZ_ASSERT(dctx); + FUZZ_ASSERT(dst); + FUZZ_ASSERT(dict); + + /* Prepare the dictionary. The data doesn't matter for decompression. */ + memset(dict, 0, dictSize); + + + /* Decompress using multiple configurations. */ + memset(&opts, 0, sizeof(opts)); + opts.stableDst = 0; + decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); + opts.stableDst = 1; + decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); + opts.stableDst = 0; + decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); + opts.stableDst = 1; + decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); + + LZ4F_freeDecompressionContext(dctx); + free(dst); + free(dict); + + return 0; +} diff --git a/ossfuzz/fuzz_helpers.h b/ossfuzz/fuzz_helpers.h index 626f209..c4a8645 100644 --- a/ossfuzz/fuzz_helpers.h +++ b/ossfuzz/fuzz_helpers.h @@ -24,8 +24,13 @@ extern "C" { #endif -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED +#include "lz4.c" /* LZ4_count, constants, mem */ +#endif + +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define FUZZ_QUOTE_IMPL(str) #str #define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str) diff --git a/ossfuzz/lz4_helpers.c b/ossfuzz/lz4_helpers.c new file mode 100644 index 0000000..9471630 --- /dev/null +++ b/ossfuzz/lz4_helpers.c @@ -0,0 +1,51 @@ +#include "fuzz_helpers.h" +#include "lz4_helpers.h" +#include "lz4hc.h" + +LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed) +{ + LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO; + info.blockSizeID = FUZZ_rand32(seed, LZ4F_max64KB - 1, LZ4F_max4MB); + if (info.blockSizeID < LZ4F_max64KB) { + info.blockSizeID = LZ4F_default; + } + info.blockMode = FUZZ_rand32(seed, LZ4F_blockLinked, LZ4F_blockIndependent); + info.contentChecksumFlag = FUZZ_rand32(seed, LZ4F_noContentChecksum, + LZ4F_contentChecksumEnabled); + info.blockChecksumFlag = FUZZ_rand32(seed, LZ4F_noBlockChecksum, + LZ4F_blockChecksumEnabled); + return info; +} + +LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed) +{ + LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; + prefs.frameInfo = FUZZ_randomFrameInfo(seed); + prefs.compressionLevel = FUZZ_rand32(seed, 0, LZ4HC_CLEVEL_MAX + 3) - 3; + prefs.autoFlush = FUZZ_rand32(seed, 0, 1); + prefs.favorDecSpeed = FUZZ_rand32(seed, 0, 1); + return prefs; +} + +size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, + const void* src, const size_t srcSize) +{ + LZ4F_decompressOptions_t opts; + memset(&opts, 0, sizeof(opts)); + opts.stableDst = 1; + LZ4F_dctx* dctx; + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + FUZZ_ASSERT(dctx); + + size_t dstSize = dstCapacity; + size_t srcConsumed = srcSize; + size_t const rc = + LZ4F_decompress(dctx, dst, &dstSize, src, &srcConsumed, &opts); + FUZZ_ASSERT(!LZ4F_isError(rc)); + FUZZ_ASSERT(rc == 0); + FUZZ_ASSERT(srcConsumed == srcSize); + + LZ4F_freeDecompressionContext(dctx); + + return dstSize; +} diff --git a/ossfuzz/lz4_helpers.h b/ossfuzz/lz4_helpers.h new file mode 100644 index 0000000..c99fb01 --- /dev/null +++ b/ossfuzz/lz4_helpers.h @@ -0,0 +1,13 @@ +#ifndef LZ4_HELPERS +#define LZ4_HELPERS + +#include "lz4frame.h" + +LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed); + +LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed); + +size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, + const void* src, const size_t srcSize); + +#endif /* LZ4_HELPERS */ diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh index a76b0d6..9782286 100755 --- a/ossfuzz/ossfuzz.sh +++ b/ossfuzz/ossfuzz.sh @@ -16,8 +16,8 @@ echo "OUT: $OUT" export MAKEFLAGS+="-j$(nproc)" pushd ossfuzz -make V=1 compress_fuzzer decompress_fuzzer +make V=1 all popd # Copy the fuzzers to the target directory. -cp -v ossfuzz/compress_fuzzer ossfuzz/decompress_fuzzer $OUT/ +cp -v ossfuzz/*_fuzzer $OUT/ diff --git a/ossfuzz/round_trip_frame_fuzzer.c b/ossfuzz/round_trip_frame_fuzzer.c new file mode 100644 index 0000000..1eea90c --- /dev/null +++ b/ossfuzz/round_trip_frame_fuzzer.c @@ -0,0 +1,39 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4frame.h" +#include "lz4_helpers.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed); + size_t const dstCapacity = LZ4F_compressFrameBound(size, &prefs); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + size_t const dstSize = + LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); + FUZZ_ASSERT(!LZ4F_isError(dstSize)); + size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/round_trip_hc_fuzzer.c b/ossfuzz/round_trip_hc_fuzzer.c new file mode 100644 index 0000000..325cdf0 --- /dev/null +++ b/ossfuzz/round_trip_hc_fuzzer.c @@ -0,0 +1,39 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4hc.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = LZ4_compressBound(size); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + int const dstSize = LZ4_compress_HC((const char*)data, dst, size, + dstCapacity, level); + FUZZ_ASSERT(dstSize > 0); + + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/round_trip_stream_fuzzer.c b/ossfuzz/round_trip_stream_fuzzer.c new file mode 100644 index 0000000..abfcd2d --- /dev/null +++ b/ossfuzz/round_trip_stream_fuzzer.c @@ -0,0 +1,302 @@ +/** + * This fuzz target performs a lz4 streaming round-trip test + * (compress & decompress), compares the result with the original, and calls + * abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" + +typedef struct { + char const* buf; + size_t size; + size_t pos; +} const_cursor_t; + +typedef struct { + char* buf; + size_t size; + size_t pos; +} cursor_t; + +typedef struct { + LZ4_stream_t* cstream; + LZ4_streamHC_t* cstreamHC; + LZ4_streamDecode_t* dstream; + const_cursor_t data; + cursor_t compressed; + cursor_t roundTrip; + uint32_t seed; + int level; +} state_t; + +cursor_t cursor_create(size_t size) +{ + cursor_t cursor; + cursor.buf = (char*)malloc(size); + cursor.size = size; + cursor.pos = 0; + FUZZ_ASSERT(cursor.buf); + return cursor; +} + +typedef void (*round_trip_t)(state_t* state); + +void cursor_free(cursor_t cursor) +{ + free(cursor.buf); +} + +state_t state_create(char const* data, size_t size, uint32_t seed) +{ + state_t state; + + state.seed = seed; + + state.data.buf = (char const*)data; + state.data.size = size; + state.data.pos = 0; + + /* Extra margin because we are streaming. */ + state.compressed = cursor_create(1024 + 2 * LZ4_compressBound(size)); + state.roundTrip = cursor_create(size); + + state.cstream = LZ4_createStream(); + FUZZ_ASSERT(state.cstream); + state.cstreamHC = LZ4_createStreamHC(); + FUZZ_ASSERT(state.cstream); + state.dstream = LZ4_createStreamDecode(); + FUZZ_ASSERT(state.dstream); + + return state; +} + +void state_free(state_t state) +{ + cursor_free(state.compressed); + cursor_free(state.roundTrip); + LZ4_freeStream(state.cstream); + LZ4_freeStreamHC(state.cstreamHC); + LZ4_freeStreamDecode(state.dstream); +} + +static void state_reset(state_t* state, uint32_t seed) +{ + state->level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + LZ4_resetStream_fast(state->cstream); + LZ4_resetStreamHC_fast(state->cstreamHC, state->level); + LZ4_setStreamDecode(state->dstream, NULL, 0); + state->data.pos = 0; + state->compressed.pos = 0; + state->roundTrip.pos = 0; + state->seed = seed; +} + +static void state_decompress(state_t* state, char const* src, int srcSize) +{ + char* dst = state->roundTrip.buf + state->roundTrip.pos; + int const dstCapacity = state->roundTrip.size - state->roundTrip.pos; + int const dSize = LZ4_decompress_safe_continue(state->dstream, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(dSize >= 0); + state->roundTrip.pos += dSize; +} + +static void state_checkRoundTrip(state_t const* state) +{ + char const* data = state->data.buf; + size_t const size = state->data.size; + FUZZ_ASSERT_MSG(size == state->roundTrip.pos, "Incorrect size!"); + FUZZ_ASSERT_MSG(!memcmp(data, state->roundTrip.buf, size), "Corruption!"); +} + +/** + * Picks a dictionary size and trims the dictionary off of the data. + * We copy the dictionary to the roundTrip so our validation passes. + */ +static size_t state_trimDict(state_t* state) +{ + /* 64 KB is the max dict size, allow slightly beyond that to test trim. */ + uint32_t maxDictSize = MIN(70 * 1024, state->data.size); + size_t const dictSize = FUZZ_rand32(&state->seed, 0, maxDictSize); + DEBUGLOG(2, "dictSize = %zu", dictSize); + FUZZ_ASSERT(state->data.pos == 0); + FUZZ_ASSERT(state->roundTrip.pos == 0); + memcpy(state->roundTrip.buf, state->data.buf, dictSize); + state->data.pos += dictSize; + state->roundTrip.pos += dictSize; + return dictSize; +} + +static void state_prefixRoundTrip(state_t* state) +{ + while (state->data.pos != state->data.size) { + char const* src = state->data.buf + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst, + srcSize, dstCapacity, 0); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } +} + +static void state_extDictRoundTrip(state_t* state) +{ + int i = 0; + cursor_t data2 = cursor_create(state->data.size); + memcpy(data2.buf, state->data.buf, state->data.size); + while (state->data.pos != state->data.size) { + char const* data = (i++ & 1) ? state->data.buf : data2.buf; + char const* src = data + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst, + srcSize, dstCapacity, 0); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } + cursor_free(data2); +} + +static void state_randomRoundTrip(state_t* state, round_trip_t rt0, + round_trip_t rt1) +{ + if (FUZZ_rand32(&state->seed, 0, 1)) { + rt0(state); + } else { + rt1(state); + } +} + +static void state_loadDictRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_loadDict(state->cstream, dict, dictSize); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip); +} + +static void state_attachDictRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_stream_t* dictStream = LZ4_createStream(); + LZ4_loadDict(dictStream, dict, dictSize); + LZ4_attach_dictionary(state->cstream, dictStream); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip); + LZ4_freeStream(dictStream); +} + +static void state_prefixHCRoundTrip(state_t* state) +{ + while (state->data.pos != state->data.size) { + char const* src = state->data.buf + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } +} + +static void state_extDictHCRoundTrip(state_t* state) +{ + int i = 0; + cursor_t data2 = cursor_create(state->data.size); + DEBUGLOG(2, "extDictHC"); + memcpy(data2.buf, state->data.buf, state->data.size); + while (state->data.pos != state->data.size) { + char const* data = (i++ & 1) ? state->data.buf : data2.buf; + char const* src = data + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(cSize > 0); + DEBUGLOG(2, "srcSize = %d", srcSize); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } + cursor_free(data2); +} + +static void state_loadDictHCRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_loadDictHC(state->cstreamHC, dict, dictSize); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixHCRoundTrip, + state_extDictHCRoundTrip); +} + +static void state_attachDictHCRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_streamHC_t* dictStream = LZ4_createStreamHC(); + LZ4_setCompressionLevel(dictStream, state->level); + LZ4_loadDictHC(dictStream, dict, dictSize); + LZ4_attach_HC_dictionary(state->cstreamHC, dictStream); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixHCRoundTrip, + state_extDictHCRoundTrip); + LZ4_freeStreamHC(dictStream); +} + +round_trip_t roundTrips[] = { + &state_prefixRoundTrip, + &state_extDictRoundTrip, + &state_loadDictRoundTrip, + &state_attachDictRoundTrip, + &state_prefixHCRoundTrip, + &state_extDictHCRoundTrip, + &state_loadDictHCRoundTrip, + &state_attachDictHCRoundTrip, +}; + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + state_t state = state_create((char const*)data, size, seed); + const int n = sizeof(roundTrips) / sizeof(round_trip_t); + int i; + + for (i = 0; i < n; ++i) { + DEBUGLOG(2, "Round trip %d", i); + state_reset(&state, seed); + roundTrips[i](&state); + state_checkRoundTrip(&state); + } + + state_free(state); + + return 0; +} diff --git a/ossfuzz/travisoss.sh b/ossfuzz/travisoss.sh index 3b2f26f..5ea884c 100755 --- a/ossfuzz/travisoss.sh +++ b/ossfuzz/travisoss.sh @@ -8,10 +8,7 @@ git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz if [[ ! -d /tmp/ossfuzz/projects/lz4 ]] then echo "Could not find the lz4 project in ossfuzz" - - # Exit with a success code while the lz4 project is not expected to exist - # on oss-fuzz. - exit 0 + exit 1 fi # Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis. |