diff options
50 files changed, 2951 insertions, 718 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c08cb2..7f03d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,9 @@ version: 2 jobs: build: working_directory: ~/lz4/lz4 + # Parallelism is broken in this file : it just plays the same tests twice. + # The script will have to be modified to support parallelism properly + # In the meantime, set it to 1. parallelism: 1 shell: /bin/bash --login # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. @@ -27,8 +30,7 @@ jobs: # To see the list of pre-built images that CircleCI provides for most common languages see # https://circleci.com/docs/2.0/circleci-images/ docker: - - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 - command: /sbin/init + - image: fbopensource/lz4-circleci-primary:0.0.4 steps: # Machine Setup # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each @@ -38,60 +40,25 @@ jobs: # In many cases you can simplify this from what is generated here. # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - # Dependencies - # This would typically go in either a build or a build-and-test job when using workflows - # Restore the dependency cache - - restore_cache: - keys: - # This branch if available - - v1-dep-{{ .Branch }}- - # Default branch if not - - v1-dep-dev- - # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v1-dep- - # This is based on your 1.0 configuration file or project settings - - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - - run: sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu - - run: sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - run: sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind - # Save dependency cache - - save_cache: - key: v1-dep-{{ .Branch }}-{{ epoch }} - paths: - # This is a broad list of cache paths to include many possible development environments - # You can probably delete some of these entries - - vendor/bundle - - ~/virtualenvs - - ~/.m2 - - ~/.ivy2 - - ~/.bundle - - ~/.go_workspace - - ~/.gradle - - ~/.cache/bower # Test # This would typically be a build job when using workflows, possibly combined with build # This is based on your 1.0 configuration file or project settings - - run: clang -v; make clangtest && make clean + - run: CFLAGS= make clangtest && make clean - run: g++ -v; make gpptest && make clean - - run: gcc -v; make c_standards && make clean - run: gcc -v; g++ -v; make ctocpptest && make clean - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean - - run: gcc-6 -v; CC=gcc-6 make c_standards && make clean - run: gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean - run: make cmake && make clean - run: make -C tests test-lz4 - run: make -C tests test-lz4c - run: make -C tests test-frametest - - run: make -C tests test-fullbench - run: make -C tests test-fuzzer && make clean - run: make -C lib all && make clean - run: pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean - run: make travis-install && make clean - run: gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean - - run: make usan && make clean - run: clang -v; make staticAnalyze && make clean - - run: make -C tests test-mem && make clean - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean - run: make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile new file mode 100644 index 0000000..7767014 --- /dev/null +++ b/.circleci/images/primary/Dockerfile @@ -0,0 +1,12 @@ +FROM circleci/buildpack-deps:bionic + +RUN sudo apt-get -y -qq update +RUN sudo apt-get -y install software-properties-common +RUN sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN sudo apt-get -y install cmake +RUN sudo apt-get -y install qemu-system-ppc qemu-user-static qemu-system-arm +RUN sudo apt-get -y install libc6-dev-armel-cross libc6-dev-arm64-cross libc6-dev-i386 +RUN sudo apt-get -y install clang clang-tools +RUN sudo apt-get -y install gcc-5 gcc-5-multilib gcc-6 +RUN sudo apt-get -y install valgrind +RUN sudo apt-get -y install gcc-multilib-powerpc-linux-gnu gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu diff --git a/.travis.yml b/.travis.yml index 2065478..bd29630 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ matrix: os: osx compiler: clang script: + - make # test library build + - make clean - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console - - CFLAGS=-m32 make -C tests clean test-lz4-contentSize + - make clean + - CFLAGS=-m32 make -C tests test-lz4-contentSize # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - name: (Precise) benchmark test @@ -24,6 +27,16 @@ matrix: script: - make -C tests test-frametest test-fuzzer + - name: ASAN tests with fuzzer and frametest + install: + - sudo sysctl -w vm.mmap_min_addr=4096 + script: + - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer + + - name: Custom LZ4_DISTANCE_MAX + script: + - MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check + - name: (Precise) g++ and clang CMake test dist: precise script: @@ -169,27 +182,39 @@ matrix: - tests/checkTag "$TRAVIS_BRANCH" - name: (Xenial) Meson + clang build - env: ALLOW_FAILURES=true + #env: ALLOW_FAILURES=true dist: xenial language: cpp compiler: clang install: - sudo apt-get install -qq python3 tree - - curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip' - && unzip ~/ninja.zip -d ~/.local/bin - - curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' - && python3 ~/get-pip.py --user - && pip3 install --user meson - script: - - meson setup - --buildtype=debug - -Db_lundef=false - -Dauto_features=enabled - -Ddefault_library=both - -Dbuild_{programs,contrib,tests,examples}=true + - | + travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' && + unzip ~/ninja.zip -d ~/.local/bin + - | + travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && + python3 ~/get-pip.py --user && + pip3 install --user meson + script: + - | + meson setup \ + --buildtype=debug \ + -Db_lundef=false \ + -Dauto_features=enabled \ + -Ddefault_library=both \ + -Dbin_programs=true \ + -Dbin_contrib=true \ + -Dbin_tests=true \ + -Dbin_examples=true \ contrib/meson build - - cd build + - pushd build - DESTDIR=./staging ninja install - tree ./staging + + # oss-fuzz compilation test + - name: Compile OSS-Fuzz targets + script: + - ./ossfuzz/travisoss.sh + allow_failures: - env: ALLOW_FAILURES=true @@ -34,6 +34,7 @@ LZ4DIR = lib PRGDIR = programs TESTDIR = tests EXDIR = examples +FUZZDIR = ossfuzz include Makefile.inc @@ -76,6 +77,7 @@ clean: @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) @$(MAKE) -C $(EXDIR) $@ > $(VOID) + @$(MAKE) -C $(FUZZDIR) $@ > $(VOID) @$(MAKE) -C contrib/gen_manual $@ > $(VOID) @$(RM) lz4$(EXT) @echo Cleaning completed @@ -125,11 +127,14 @@ test: $(MAKE) -C $(TESTDIR) $@ $(MAKE) -C $(EXDIR) $@ +clangtest: CFLAGS ?= -O3 +clangtest: CFLAGS += -Werror -Wconversion -Wno-sign-conversion +clangtest: CC = clang clangtest: clean - clang -v - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(LZ4DIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) all CC=clang - @CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) all CC=clang + $(CC) -v + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(LZ4DIR) all CC=$(CC) + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(PRGDIR) all CC=$(CC) + @CFLAGS="$(CFLAGS)" $(MAKE) -C $(TESTDIR) all CC=$(CC) clangtest-native: clean clang -v diff --git a/contrib/cmake_unofficial/CMakeLists.txt b/contrib/cmake_unofficial/CMakeLists.txt index b09c4fb..42d92ea 100644 --- a/contrib/cmake_unofficial/CMakeLists.txt +++ b/contrib/cmake_unofficial/CMakeLists.txt @@ -172,6 +172,7 @@ if(NOT LZ4_BUNDLED_MODE) include(GNUInstallDirs) install(TARGETS ${LZ4_PROGRAMS_BUILT} + BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}" RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") install(TARGETS ${LZ4_LIBRARIES_BUILT} LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" diff --git a/contrib/meson/README.md b/contrib/meson/README.md index fa18493..a44850a 100644 --- a/contrib/meson/README.md +++ b/contrib/meson/README.md @@ -13,7 +13,7 @@ This Meson build system is provided with no guarantee. `cd` to this meson directory (`contrib/meson`) ```sh -meson setup --buildtype=release -Ddefault_library=shared -Dbuild_programs=true builddir +meson setup --buildtype=release -Ddefault_library=shared -Dbin_programs=true builddir cd builddir ninja # to build ninja install # to install diff --git a/contrib/meson/contrib/gen_manual/meson.build b/contrib/meson/contrib/gen_manual/meson.build index 6233cdc..38180e9 100644 --- a/contrib/meson/contrib/gen_manual/meson.build +++ b/contrib/meson/contrib/gen_manual/meson.build @@ -22,6 +22,7 @@ gen_manual = executable('gen_manual', join_paths(lz4_root_dir, 'contrib/gen_manual/gen_manual.cpp'), cpp_args: gen_manual_cppflags, include_directories: gen_manual_includes, + native: true, install: false) # Update lz4 manual diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index bf30eae..65a4c26 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -11,7 +11,7 @@ project('lz4', ['c'], license: ['BSD', 'GPLv2'], default_options : ['c_std=c99', 'buildtype=release'], - version: '1.8.3', + version: 'DUMMY', meson_version: '>=0.47.0') cc = meson.get_compiler('c') @@ -38,13 +38,10 @@ lz4_h_file = join_paths(meson.current_source_dir(), '../../lib/lz4.h') GetLz4LibraryVersion_py = files('GetLz4LibraryVersion.py') r = run_command(python3, GetLz4LibraryVersion_py, lz4_h_file) if r.returncode() == 0 - output = r.stdout().strip() - if output.version_compare('>@0@'.format(lz4_version)) - lz4_version = output - message('Project version is now: @0@'.format(lz4_version)) - endif + lz4_version = r.stdout().strip() + message('Project version is now: @0@'.format(lz4_version)) else - warning('Cannot find project version in @0@'.format(lz4_h_file)) + error('Cannot find project version in @0@'.format(lz4_h_file)) endif lz4_libversion = lz4_version @@ -72,17 +69,17 @@ use_debug = get_option('debug') debug_level = get_option('debug_level') use_backtrace = get_option('backtrace') -build_programs = get_option('build_programs') -build_contrib = get_option('build_contrib') -build_tests = get_option('build_tests') -build_examples = get_option('build_examples') +bin_programs = get_option('bin_programs') +bin_contrib = get_option('bin_contrib') +bin_tests = get_option('bin_tests') +bin_examples = get_option('bin_examples') #feature_multi_thread = get_option('multi_thread') # ============================================================================= # Dependencies # ============================================================================= -#libm_dep = cc.find_library('m', required: build_tests) +#libm_dep = cc.find_library('m', required: bin_tests) #thread_dep = dependency('threads', required: feature_multi_thread) #use_multi_thread = thread_dep.found() @@ -111,18 +108,18 @@ endif subdir('lib') -if build_programs +if bin_programs subdir('programs') endif -if build_tests +if bin_tests subdir('tests') endif -if build_contrib +if bin_contrib subdir('contrib') endif -if build_examples +if bin_examples subdir('examples') endif diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt index f6a4ae7..a409c2d 100644 --- a/contrib/meson/meson_options.txt +++ b/contrib/meson/meson_options.txt @@ -14,11 +14,11 @@ option('debug_level', type: 'integer', min: 0, max: 7, value: 1, option('backtrace', type: 'boolean', value: false, description: 'Display a stack backtrace when execution generates a runtime exception') -option('build_programs', type: 'boolean', value: false, +option('bin_programs', type: 'boolean', value: false, description: 'Enable programs build') -option('build_tests', type: 'boolean', value: false, +option('bin_tests', type: 'boolean', value: false, description: 'Enable tests build') -option('build_contrib', type: 'boolean', value: false, +option('bin_contrib', type: 'boolean', value: false, description: 'Enable contrib build') -option('build_examples', type: 'boolean', value: false, +option('bin_examples', type: 'boolean', value: false, description: 'Enable examples build') diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index 3a9e0db..a477584 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -1,10 +1,10 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>1.9.1 Manual</title> +<title>1.9.2 Manual</title> </head> <body> -<h1>1.9.1 Manual</h1> +<h1>1.9.2 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> @@ -21,7 +21,7 @@ </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> - LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. @@ -33,16 +33,19 @@ - unbounded multiple steps (described as Streaming compression) lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). - Decompressing a block requires additional metadata, such as its compressed size. + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. Each application is free to encode and pass such metadata in whichever way it wants. lz4.h only handle blocks, it can not generate Frames. Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames. <BR></pre> <a name="Chapter2"></a><h2>Version</h2><pre></pre> @@ -66,27 +69,35 @@ <a name="Chapter4"></a><h2>Simple Functions</h2><pre></pre> <pre><b>int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); -</b><p> Compresses 'srcSize' bytes from buffer 'src' - into already allocated 'dst' buffer of size 'dstCapacity'. - Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a more limited 'dst' budget, - compression stops *immediately*, and the function result is zero. - In which case, 'dst' content is undefined (invalid). - srcSize : max supported value is LZ4_MAX_INPUT_SIZE. - dstCapacity : size of buffer 'dst' (which must be already allocated) - @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails - Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). +</b><p> Compresses 'srcSize' bytes from buffer 'src' + into already allocated 'dst' buffer of size 'dstCapacity'. + Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'src' into a more limited 'dst' budget, + compression stops *immediately*, and the function result is zero. + In which case, 'dst' content is undefined (invalid). + srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + dstCapacity : size of buffer 'dst' (which must be already allocated) + @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + or 0 if compression fails + Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + </p></pre><BR> <pre><b>int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); -</b><p> compressedSize : is the exact complete size of the compressed block. - dstCapacity : is the size of destination buffer, which must be already allocated. - @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - If destination buffer is not large enough, decoding will stop and output an error code (negative value). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). +</b><p> compressedSize : is the exact complete size of the compressed block. + dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + If destination buffer is not large enough, decoding will stop and output an error code (negative value). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Note 1 : This function is protected against malicious data packets : + it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + even if the compressed block is maliciously modified to order the decoder to do these actions. + In such case, the decoder stops immediately, and considers the compressed block malformed. + Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + The implementation is free to send / store / derive this information in whichever way is most beneficial. + If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + </p></pre><BR> <a name="Chapter5"></a><h2>Advanced Functions</h2><pre></pre> @@ -357,6 +368,61 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); </p></pre><BR> +<pre><b></b><p> + It's possible to have input and output sharing the same buffer, + for highly contrained memory environments. + In both cases, it requires input to lay at the end of the buffer, + and decompression to start at beginning of the buffer. + Buffer size must feature some margin, hence be larger than final size. + + |<------------------------buffer--------------------------------->| + |<-----------compressed data--------->| + |<-----------decompressed size------------------>| + |<----margin---->| + + This technique is more useful for decompression, + since decompressed size is typically larger, + and margin is short. + + In-place decompression will work inside any buffer + which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + This presumes that decompressedSize > compressedSize. + Otherwise, it means compression actually expanded data, + and it would be more efficient to store such data with a flag indicating it's not compressed. + This can happen when data is not compressible (already compressed, or encrypted). + + For in-place compression, margin is larger, as it must be able to cope with both + history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + and data expansion, which can happen when input is not compressible. + As a consequence, buffer size requirements are much higher, + and memory savings offered by in-place compression are more limited. + + There are ways to limit this cost for compression : + - Reduce history size, by modifying LZ4_DISTANCE_MAX. + Note that it is a compile-time constant, so all compressions will apply this limit. + Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + so it's a reasonable trick when inputs are known to be small. + - Require the compressor to deliver a "maximum compressed size". + This is the `dstCapacity` parameter in `LZ4_compress*()`. + When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + in which case, the return code will be 0 (zero). + The caller must be ready for these cases to happen, + and typically design a backup scheme to send data uncompressed. + The combination of both techniques can significantly reduce + the amount of margin required for in-place compression. + + In-place compression can work in any buffer + which size is >= (maxCompressedSize) + with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + so it's possible to reduce memory requirements by playing with them. + +</p></pre><BR> + +<pre><b>#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) </b>/**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */<b> +</b></pre><BR> +<pre><b>#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) </b>/**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */<b> +</b></pre><BR> <a name="Chapter9"></a><h2>PRIVATE DEFINITIONS</h2><pre> Do not use these definitions directly. They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 2cf4f03..72f27c8 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -1,10 +1,10 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>1.9.1 Manual</title> +<title>1.9.2 Manual</title> </head> <body> -<h1>1.9.1 Manual</h1> +<h1>1.9.2 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c index a878577..bc8391e 100644 --- a/examples/HCStreaming_ringBuffer.c +++ b/examples/HCStreaming_ringBuffer.c @@ -26,6 +26,7 @@ #include <stdint.h> #include <stdlib.h> #include <string.h> +#include <assert.h> enum { MESSAGE_MAX_BYTES = 1024, @@ -39,7 +40,8 @@ size_t write_int32(FILE* fp, int32_t i) { } size_t write_bin(FILE* fp, const void* array, int arrayBytes) { - return fwrite(array, 1, arrayBytes, fp); + assert(arrayBytes >= 0); + return fwrite(array, 1, (size_t)arrayBytes, fp); } size_t read_int32(FILE* fp, int32_t* i) { @@ -47,7 +49,8 @@ size_t read_int32(FILE* fp, int32_t* i) { } size_t read_bin(FILE* fp, void* array, int arrayBytes) { - return fread(array, 1, arrayBytes, fp); + assert(arrayBytes >= 0); + return fread(array, 1, (size_t)arrayBytes, fp); } @@ -174,7 +177,7 @@ int main(int argc, const char** argv) return 0; } - if (!strcmp(argv[1], "-p")) pause = 1, fileID = 2; + if (!strcmp(argv[1], "-p")) { pause = 1; fileID = 2; } snprintf(inpFilename, 256, "%s", argv[fileID]); snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9); diff --git a/examples/blockStreaming_doubleBuffer.md b/examples/blockStreaming_doubleBuffer.md index 3027ea3..38dc2e8 100644 --- a/examples/blockStreaming_doubleBuffer.md +++ b/examples/blockStreaming_doubleBuffer.md @@ -1,7 +1,7 @@ # LZ4 Streaming API Example : Double Buffer by *Takayuki Matsuoka* -`blockStreaming_doubleBuffer.c` is LZ4 Straming API example which implements double buffer (de)compression. +`blockStreaming_doubleBuffer.c` is LZ4 Streaming API example which implements double buffer (de)compression. Please note : @@ -76,10 +76,10 @@ so it just compress the line without dependencies and generates compressed block After that, write {Out#1} to the file. Next, read second block to double buffer's second page. And compress it. -In this time, LZ4 can use dependency to Block#1 to improve compression ratio. +This time, LZ4 can use dependency to Block#1 to improve compression ratio. This dependency is called "Prefix mode". -Next, read third block to double buffer's *first* page. And compress it. +Next, read third block to double buffer's *first* page, and compress it. Also this time, LZ4 can use dependency to Block#2. This dependency is called "External Dictonaly mode". diff --git a/examples/simple_buffer.c b/examples/simple_buffer.c index 54e542a..6afc62a 100644 --- a/examples/simple_buffer.c +++ b/examples/simple_buffer.c @@ -3,18 +3,18 @@ * Copyright : Kyle Harper * License : Follows same licensing as the lz4.c/lz4.h program at any given time. Currently, BSD 2. * Description: Example program to demonstrate the basic usage of the compress/decompress functions within lz4.c/lz4.h. - * The functions you'll likely want are LZ4_compress_default and LZ4_decompress_safe. Both of these are documented in - * the lz4.h header file; I recommend reading them. + * The functions you'll likely want are LZ4_compress_default and LZ4_decompress_safe. + * Both of these are documented in the lz4.h header file; I recommend reading them. */ -/* Includes, for Power! */ -#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression. +/* Dependencies */ #include <stdio.h> // For printf() #include <string.h> // For memcmp() #include <stdlib.h> // For exit() +#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression. /* - * Easy show-error-and-bail function. + * Simple show-error-and-bail function. */ void run_screaming(const char* message, const int code) { printf("%s \n", message); @@ -32,19 +32,19 @@ int main(void) { // 1) The return codes of LZ4_ functions are important. // Read lz4.h if you're unsure what a given code means. // 2) LZ4 uses char* pointers in all LZ4_ functions. - // This is baked into the API and probably not going to change. - // If your program uses pointers that are unsigned char*, void*, or otherwise different, - // you may need to do some casting or set the right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). + // This is baked into the API and not going to change, for consistency. + // If your program uses different pointer types, + // you may need to do some casting or set the right -Wno compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). /* Compression */ // We'll store some text into a variable pointed to by *src to be compressed later. - const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat."; // The compression function needs to know how many bytes exist. Since we're using a string, we can use strlen() + 1 (for \0). const int src_size = (int)(strlen(src) + 1); // LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound(). const int max_dst_size = LZ4_compressBound(src_size); // We will use that size for our destination boundary when allocating space. - char* compressed_data = malloc(max_dst_size); + char* compressed_data = malloc((size_t)max_dst_size); if (compressed_data == NULL) run_screaming("Failed to allocate memory for *compressed_data.", 1); // That's all the information and preparation LZ4 needs to compress *src into *compressed_data. @@ -52,21 +52,27 @@ int main(void) { // Save the return value for error checking. const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size); // Check return_value to determine what happened. - if (compressed_data_size < 0) - run_screaming("A negative result from LZ4_compress_default indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", compressed_data_size); - if (compressed_data_size == 0) - run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); + if (compressed_data_size <= 0) + run_screaming("A 0 or negative result from LZ4_compress_default() indicates a failure trying to compress the data. ", 1); if (compressed_data_size > 0) - printf("We successfully compressed some data!\n"); + printf("We successfully compressed some data! Ratio: %.2f\n", + (float) compressed_data_size/src_size); // Not only does a positive return_value mean success, the value returned == the number of bytes required. // You can use this to realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. - compressed_data = (char *)realloc(compressed_data, compressed_data_size); + compressed_data = (char *)realloc(compressed_data, (size_t)compressed_data_size); if (compressed_data == NULL) run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); + /* Decompression */ - // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a - // *new_src location of size src_size since we know that value. + // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! + // The decompression will need to know the compressed size, and an upper bound of the decompressed size. + // In this example, we just re-use this information from previous section, + // but in a real-world scenario, metadata must be transmitted to the decompression side. + // Each implementation is in charge of this part. Oftentimes, it adds some header of its own. + // Sometimes, the metadata can be extracted from the local context. + + // First, let's create a *new_src location of size src_size since we know that value. char* const regen_buffer = malloc(src_size); if (regen_buffer == NULL) run_screaming("Failed to allocate memory for *regen_buffer.", 1); @@ -77,17 +83,17 @@ int main(void) { free(compressed_data); /* no longer useful */ if (decompressed_size < 0) run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", decompressed_size); - if (decompressed_size == 0) - run_screaming("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.", 1); - if (decompressed_size > 0) + if (decompressed_size >= 0) printf("We successfully decompressed some data!\n"); // Not only does a positive return value mean success, // value returned == number of bytes regenerated from compressed_data stream. + if (decompressed_size != src_size) + run_screaming("Decompressed data is different from original! \n", 1); /* Validation */ // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. if (memcmp(src, regen_buffer, src_size) != 0) run_screaming("Validation failed. *src and *new_src are not identical.", 1); - printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer); + printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer); return 0; } diff --git a/lib/Makefile b/lib/Makefile index 330642a..8f21d3d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -55,10 +55,11 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) SRCFILES := $(sort $(wildcard *.c)) +include ../Makefile.inc # OS X linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(OS), Darwin) +ifeq ($(TARGET_OS), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) @@ -70,8 +71,6 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif -include ../Makefile.inc - .PHONY: default default: lib-release diff --git a/lib/README.md b/lib/README.md index b753195..cba2c34 100644 --- a/lib/README.md +++ b/lib/README.md @@ -56,8 +56,8 @@ The following build macro can be selected at compilation time : - `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow. Set to 65535 by default, which is the maximum value supported by lz4 format. Reducing maximum distance will reduce opportunities for LZ4 to find matches, - hence will produce worse the compression ratio. - However, a smaller max distance may allow compatibility with specific decoders using limited memory budget. + hence will produce a worse compression ratio. + However, a smaller max distance can allow compatibility with specific decoders using limited memory budget. This build macro only influences the compressed output of the compressor. - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning. @@ -74,9 +74,7 @@ The following build macro can be selected at compilation time : lz4 source code can be amalgamated into a single file. One can combine all source code into `lz4_all.c` by using following command: ``` -cat lz4.c > lz4_all.c -cat lz4hc.c >> lz4_all.c -cat lz4frame.c >> lz4_all.c +cat lz4.c lz4hc.c lz4frame.c > lz4_all.c ``` (`cat` file order is important) then compile `lz4_all.c`. All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`. @@ -106,6 +106,7 @@ #define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ #endif +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ @@ -183,6 +184,60 @@ /*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<<ML_BITS)-1) +#define RUN_BITS (8-ML_BITS) +#define RUN_MASK ((1U<<RUN_BITS)-1) + + +/*-************************************ +* Error detection +**************************************/ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include <assert.h> +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include <stdio.h> +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************ * Types **************************************/ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) @@ -317,6 +372,11 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #ifndef LZ4_FAST_DEC_LOOP # if defined(__i386__) || defined(__x86_64__) # define LZ4_FAST_DEC_LOOP 1 +# elif defined(__aarch64__) && !defined(__clang__) + /* On aarch64, we disable this optimization for clang because on certain + * mobile chipsets and clang, it reduces performance. For more information + * refer to https://github.com/lz4/lz4/pull/707. */ +# define LZ4_FAST_DEC_LOOP 1 # else # define LZ4_FAST_DEC_LOOP 0 # endif @@ -358,29 +418,35 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e); } +/* LZ4_memcpy_using_offset() presumes : + * - dstEnd >= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ LZ4_FORCE_O2_INLINE_GCC_PPC64LE void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */ + switch(offset) { case 1: memset(v, *srcPtr, 8); - goto copy_loop; + break; case 2: memcpy(v, srcPtr, 2); memcpy(&v[2], srcPtr, 2); memcpy(&v[4], &v[0], 4); - goto copy_loop; + break; case 4: memcpy(v, srcPtr, 4); memcpy(&v[4], srcPtr, 4); - goto copy_loop; + break; default: LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); return; } - copy_loop: memcpy(dstPtr, v, 8); dstPtr += 8; while (dstPtr < dstEnd) { @@ -392,63 +458,6 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si /*-************************************ -* Common Constants -**************************************/ -#define MINMATCH 4 - -#define WILDCOPYLENGTH 8 -#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ -#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ -#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ -#define FASTLOOP_SAFE_DISTANCE 64 -static const int LZ4_minLength = (MFLIMIT+1); - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#ifndef LZ4_DISTANCE_MAX /* can be user - defined at compile time */ -# define LZ4_DISTANCE_MAX 65535 -#endif - -#if (LZ4_DISTANCE_MAX > 65535) /* max supported by LZ4 format */ -# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" -#endif - -#define ML_BITS 4 -#define ML_MASK ((1U<<ML_BITS)-1) -#define RUN_BITS (8-ML_BITS) -#define RUN_MASK ((1U<<RUN_BITS)-1) - - -/*-************************************ -* Error detection -**************************************/ -#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) -# include <assert.h> -#else -# ifndef assert -# define assert(condition) ((void)0) -# endif -#endif - -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ - -#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) -# include <stdio.h> -static int g_debuglog_enable = 1; -# define DEBUGLOG(l, ...) { \ - if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif - - -/*-************************************ * Common functions **************************************/ static unsigned LZ4_NbCommonBytes (reg_t val) @@ -460,7 +469,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward64( &r, (U64)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + return (unsigned)__builtin_ctzll((U64)val) >> 3; # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, @@ -478,7 +487,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); + return (unsigned)__builtin_ctz((U32)val) >> 3; # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, @@ -494,7 +503,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); + return (unsigned)__builtin_clzll((U64)val) >> 3; # else static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. @@ -511,7 +520,7 @@ static unsigned LZ4_NbCommonBytes (reg_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); + return (unsigned)__builtin_clz((U32)val) >> 3; # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -606,9 +615,11 @@ int LZ4_sizeofState() { return LZ4_STREAMSIZE; } extern "C" { #endif -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); -int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize); +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); #if defined (__cplusplus) } @@ -643,6 +654,18 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } +static void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + static void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) { switch (tableType) @@ -703,18 +726,19 @@ static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, - const void* tableBase, tableType_t tableType, - const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } -LZ4_FORCE_INLINE void LZ4_prepareTable( - LZ4_stream_t_internal* const cctx, - const int inputSize, - const tableType_t tableType) { +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { /* If compression failed during the previous step, then the context * is marked as dirty, therefore, it has to be fully reset. */ @@ -729,9 +753,10 @@ LZ4_FORCE_INLINE void LZ4_prepareTable( * out if it's safe to leave as is or whether it needs to be reset. */ if (cctx->tableType != clearedTable) { + assert(inputSize >= 0); if (cctx->tableType != tableType - || (tableType == byU16 && cctx->currentOffset + inputSize >= 0xFFFFU) - || (tableType == byU32 && cctx->currentOffset > 1 GB) + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) || tableType == byPtr || inputSize >= 4 KB) { @@ -811,9 +836,9 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, tableType=%u", inputSize, tableType); /* If init conditions are not met, we don't have to mark stream * as having dirty context, since no action was taken yet */ - if (outputDirective == fillOutput && maxOutputSize < 1) return 0; /* Impossible to store anything */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported inputSize, too large (or negative) */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ assert(acceleration >= 1); @@ -841,6 +866,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( for ( ; ; ) { const BYTE* match; BYTE* token; + const BYTE* filledIp; /* Find a match */ if (tableType == byPtr) { @@ -909,10 +935,14 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); - if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) continue; /* match outside of valid area */ + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ assert(matchIndex < current); - if ((tableType != byU16) && (matchIndex+LZ4_DISTANCE_MAX < current)) continue; /* too far */ - if (tableType == byU16) assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* too_far presumed impossible with byU16 */ + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ if (LZ4_read32(match) == LZ4_read32(ip)) { if (maybe_extMem) offset = current - matchIndex; @@ -923,15 +953,16 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( } /* Catch up */ + filledIp = ip; while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ - + } if ((outputDirective == fillOutput) && (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { op--; @@ -1002,12 +1033,26 @@ _next_match: } if ((outputDirective) && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) { + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { if (outputDirective == fillOutput) { /* Match description too long : reduce it */ - U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 2 - 1 - LASTLITERALS) * 255; + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } } else { assert(outputDirective == limitedOutput); return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ @@ -1027,6 +1072,8 @@ _next_match: } else *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; @@ -1076,7 +1123,7 @@ _next_match: LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); assert(matchIndex < current); if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) - && ((tableType==byU16) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) && (LZ4_read32(match) == LZ4_read32(ip)) ) { token=op++; *token=0; @@ -1143,7 +1190,7 @@ int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } } else { - if (inputSize < LZ4_64Klimit) {; + if (inputSize < LZ4_64Klimit) { return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); } else { const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; @@ -1306,12 +1353,12 @@ static size_t LZ4_stream_t_alignment(void) LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); - if (buffer == NULL) return NULL; - if (size < sizeof(LZ4_stream_t)) return NULL; + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } #ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : it reports an aligment of 8-bytes, while actually aligning LZ4_stream_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) return NULL; /* alignment check */ + if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ #endif MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); return (LZ4_stream_t*)buffer; @@ -1361,18 +1408,18 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) * there are only valid offsets in the window, which allows an optimization * in LZ4_compress_fast_continue() where it uses noDictIssue even when the * dictionary isn't a full 64k. */ - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - base = dictEnd - 64 KB - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); dict->currentOffset += 64 KB; - dict->tableType = tableType; if (dictSize < (int)HASH_UNIT) { return 0; } + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + base = dictEnd - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = tableType; + while (p <= dictEnd-HASH_UNIT) { LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; @@ -1381,26 +1428,37 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return (int)dict->dictSize; } -void LZ4_attach_dictionary(LZ4_stream_t *working_stream, const LZ4_stream_t *dictionary_stream) { +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { + const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + /* Calling LZ4_resetStream_fast() here makes sure that changes will not be * erased by subsequent calls to LZ4_resetStream_fast() in case stream was * marked as having dirty context, e.g. requiring full reset. */ - LZ4_resetStream_fast(working_stream); + LZ4_resetStream_fast(workingStream); - if (dictionary_stream != NULL) { + if (dictCtx != NULL) { /* If the current offset is zero, we will never look in the * external dictionary context, since there is no value a table * entry can take that indicate a miss. In that case, we need * to bump the offset to something non-zero. */ - if (working_stream->internal_donotuse.currentOffset == 0) { - working_stream->internal_donotuse.currentOffset = 64 KB; + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; } - working_stream->internal_donotuse.dictCtx = &(dictionary_stream->internal_donotuse); - } else { - working_stream->internal_donotuse.dictCtx = NULL; } + workingStream->internal_donotuse.dictCtx = dictCtx; } @@ -1435,7 +1493,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); - if (streamPtr->dirty) return 0; /* Uninitialized structure detected */ + if (streamPtr->dirty) { return 0; } /* Uninitialized structure detected */ LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; @@ -1532,8 +1590,8 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = (int)dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } memmove(safeBuffer, previousDictEnd - dictSize, dictSize); @@ -1607,7 +1665,7 @@ LZ4_decompress_generic( const size_t dictSize /* note : = 0 if noDict */ ) { - if (src == NULL) return -1; + if (src == NULL) { return -1; } { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; @@ -1636,9 +1694,13 @@ LZ4_decompress_generic( /* Special cases */ assert(lowPrefix <= op); - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); - if ((endOnInput) && unlikely(srcSize==0)) return -1; + if ((endOnInput) && (unlikely(outputSize==0))) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); } + if ((endOnInput) && unlikely(srcSize==0)) { return -1; } /* Currently the fast loop shows a regression on qualcomm arm chips. */ #if LZ4_FAST_DEC_LOOP @@ -1651,7 +1713,7 @@ LZ4_decompress_generic( while (1) { /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ assert(oend - op >= FASTLOOP_SAFE_DISTANCE); - if (endOnInput) assert(ip < iend); + if (endOnInput) { assert(ip < iend); } token = *ip++; length = token >> ML_BITS; /* literal length */ @@ -1661,18 +1723,18 @@ LZ4_decompress_generic( if (length == RUN_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ /* copy literals */ cpy = op+length; LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if (endOnInput) { /* LZ4_decompress_safe() */ - if ((cpy>oend-32) || (ip+length>iend-32)) goto safe_literal_copy; + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } LZ4_wildCopy32(op, ip, cpy); } else { /* LZ4_decompress_fast() */ - if (cpy>oend-8) goto safe_literal_copy; + if (cpy>oend-8) { goto safe_literal_copy; } LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and only relies on end-of-block properties */ } @@ -1682,14 +1744,14 @@ LZ4_decompress_generic( if (endOnInput) { /* LZ4_decompress_safe() */ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); /* We don't need to check oend, since we check it once for each loop below */ - if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) goto safe_literal_copy; + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ memcpy(op, ip, 16); } else { /* LZ4_decompress_fast() */ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : * it doesn't know input length, and relies on end-of-block properties */ memcpy(op, ip, 8); - if (length > 8) memcpy(op+8, ip+8, 8); + if (length > 8) { memcpy(op+8, ip+8, 8); } } ip += length; op = cpy; } @@ -1697,17 +1759,17 @@ LZ4_decompress_generic( /* get offset */ offset = LZ4_readLE16(ip); ip+=2; match = op - offset; + assert(match <= op); /* get matchlength */ length = token & ML_MASK; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (length == ML_MASK) { variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); - if (error != ok) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ length += MINMATCH; if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { goto safe_match_copy; @@ -1719,8 +1781,12 @@ LZ4_decompress_generic( } /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ - if (!(dict == usingExtDict) || (match >= lowPrefix)) { + if ((dict == withPrefix64k) || (match >= lowPrefix)) { if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + memcpy(op, match, 8); memcpy(op+8, match+8, 8); memcpy(op+16, match+16, 2); @@ -1728,12 +1794,15 @@ LZ4_decompress_generic( continue; } } } + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { - if (partialDecoding) length = MIN(length, (size_t)(oend-op)); - else goto _output_error; /* doesn't respect parsing restriction */ - } + if (partialDecoding) { + length = MIN(length, (size_t)(oend-op)); /* reach end of buffer */ + } else { + goto _output_error; /* end-of-block condition violated */ + } } if (length <= (size_t)(lowPrefix-match)) { /* match fits entirely within external dictionary : just copy */ @@ -1748,7 +1817,7 @@ LZ4_decompress_generic( if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ BYTE* const endOfMatch = op + restSize; const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + while (op < endOfMatch) { *op++ = *copyFrom++; } } else { memcpy(op, lowPrefix, restSize); op += restSize; @@ -1821,11 +1890,11 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { - variable_length_error error = ok; - length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); - if (error == initial_error) goto _output_error; - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ - if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ } /* copy literals */ @@ -1837,21 +1906,50 @@ LZ4_decompress_generic( if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * If we've hit the input parsing condition then this must be the last sequence. + * If we've hit the output parsing condition then we are either using partialDecoding + * or we've hit the output parsing condition. + */ if (partialDecoding) { - if (cpy > oend) { cpy = oend; assert(op<=oend); length = (size_t)(oend-op); } /* Partial decoding : stop in the middle of literal segment */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + /* If we're in this block because of the input parsing condition, then we must be on the + * last sequence (or invalid), so we must check that we exactly consume the input. + */ + if ((ip+length>iend-(2+1+LASTLITERALS)) && (ip+length != iend)) { goto _output_error; } + assert(ip+length <= iend); + /* We are finishing in the middle of a literals segment. + * Break after the copy. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + assert(ip+length <= iend); } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { goto _output_error; } } - memcpy(op, ip, length); + memmove(op, ip, length); /* supports overlapping memory regions, which only matters for in-place decompression scenarios */ ip += length; op += length; - if (!partialDecoding || (cpy == oend)) { - /* Necessarily EOF, due to parsing restrictions */ + /* Necessarily EOF when !partialDecoding. When partialDecoding + * it is EOF if we've either filled the output buffer or hit + * the input parsing restriction. + */ + if (!partialDecoding || (cpy == oend) || (ip == iend)) { break; } - } else { LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ ip += length; op = cpy; @@ -1865,13 +1963,6 @@ LZ4_decompress_generic( length = token & ML_MASK; _copy_match: - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - if (!partialDecoding) { - assert(oend > op); - assert(oend - op >= 4); - LZ4_write32(op, 0); /* silence an msan warning when offset==0; costs <1%; */ - } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ - if (length == ML_MASK) { variable_length_error error = ok; length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); @@ -1883,6 +1974,7 @@ LZ4_decompress_generic( #if LZ4_FAST_DEC_LOOP safe_match_copy: #endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { @@ -1910,6 +2002,7 @@ LZ4_decompress_generic( } } continue; } + assert(match >= lowPrefix); /* copy match within block */ cpy = op + length; @@ -1921,16 +2014,17 @@ LZ4_decompress_generic( const BYTE* const matchEnd = match + mlen; BYTE* const copyEnd = op + mlen; if (matchEnd > op) { /* overlap copy */ - while (op < copyEnd) *op++ = *match++; + while (op < copyEnd) { *op++ = *match++; } } else { memcpy(op, match, mlen); } op = copyEnd; - if (op==oend) break; + if (op == oend) { break; } continue; } if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; @@ -1946,25 +2040,26 @@ LZ4_decompress_generic( if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ if (op < oCopyLimit) { LZ4_wildCopy8(op, match, oCopyLimit); match += oCopyLimit - op; op = oCopyLimit; } - while (op < cpy) *op++ = *match++; + while (op < cpy) { *op++ = *match++; } } else { memcpy(op, match, 8); - if (length > 16) LZ4_wildCopy8(op+8, match+8, cpy); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } } op = cpy; /* wildcopy correction */ } /* end of decoding */ - if (endOnInput) + if (endOnInput) { return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else + } else { return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } /* Overflow error detected */ _output_error: @@ -2079,7 +2174,7 @@ LZ4_streamDecode_t* LZ4_createStreamDecode(void) int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (LZ4_stream == NULL) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } @@ -2214,18 +2309,22 @@ int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressed if (dictSize==0) return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= 64 KB - 1) + if (dictSize >= 64 KB - 1) { return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); - return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } - return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { if (dictSize==0 || dictStart+dictSize == dest) return LZ4_decompress_fast(source, dest, originalSize); - return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -2237,9 +2336,9 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress(const char* src, char* dest, int srcSize) { - return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { @@ -46,7 +46,7 @@ extern "C" { /** Introduction - LZ4 is lossless compression algorithm, providing compression speed at 500 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. @@ -58,16 +58,19 @@ extern "C" { - unbounded multiple steps (described as Streaming compression) lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). - Decompressing a block requires additional metadata, such as its compressed size. + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. Each application is free to encode and pass such metadata in whichever way it wants. lz4.h only handle blocks, it can not generate Frames. Blocks are different from Frames (doc/lz4_Frame_format.md). Frames bundle both blocks and metadata in a specified manner. - This are required for compressed data to be self-contained and portable. + Embedding metadata is required for compressed data to be self-contained and portable. Frame format is delivered through a companion API, declared in lz4frame.h. - Note that the `lz4` CLI can only manage frames. + The `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -97,7 +100,7 @@ extern "C" { /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -129,29 +132,35 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'srcSize' bytes from buffer 'src' - into already allocated 'dst' buffer of size 'dstCapacity'. - Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a more limited 'dst' budget, - compression stops *immediately*, and the function result is zero. - In which case, 'dst' content is undefined (invalid). - srcSize : max supported value is LZ4_MAX_INPUT_SIZE. - dstCapacity : size of buffer 'dst' (which must be already allocated) - @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails - Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). -*/ + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the exact complete size of the compressed block. - dstCapacity : is the size of destination buffer, which must be already allocated. - @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - If destination buffer is not large enough, decoding will stop and output an error code (negative value). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Note : This function is protected against malicious data packets (never writes outside 'dst' buffer, nor read outside 'source' buffer). -*/ + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); @@ -388,6 +397,8 @@ LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecod */ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); +#endif /* LZ4_H_2983827168210 */ + /*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! @@ -413,14 +424,17 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. ******************************************************************************/ +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + #ifdef LZ4_PUBLISH_STATIC_FUNCTIONS #define LZ4LIB_STATIC_API LZ4LIB_API #else #define LZ4LIB_STATIC_API #endif -#ifdef LZ4_STATIC_LINKING_ONLY - /*! LZ4_compress_fast_extState_fastReset() : * A variant of LZ4_compress_fast_extState(). @@ -462,8 +476,75 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c */ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 /*-************************************************************ * PRIVATE DEFINITIONS @@ -567,6 +648,7 @@ union LZ4_streamDecode_u { } ; /* previously typedef'd to LZ4_streamDecode_t */ + /*-************************************ * Obsolete Functions **************************************/ @@ -601,8 +683,8 @@ union LZ4_streamDecode_u { #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ /* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); @@ -674,7 +756,7 @@ LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int or LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -#endif /* LZ4_H_2983827168210 */ +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a10e4af..c9f630d 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -213,8 +213,8 @@ static void LZ4F_writeLE64 (void* dst, U64 value64) static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ -static const size_t BHSize = 4; /* block header : size, and compress flag */ -static const size_t BFSize = 4; /* block footer : checksum (optional) */ +static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */ +static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */ /*-************************************ @@ -327,6 +327,7 @@ static size_t LZ4F_compressBound_internal(size_t srcSize, { LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; U32 const flush = prefsPtr->autoFlush | (srcSize==0); LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; @@ -1130,8 +1131,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize } /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif dctx->frameInfo.frameType = LZ4F_frame; /* Flags */ @@ -1170,10 +1173,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize /* check header */ assert(frameHeaderSize > 5); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); if (HC != srcPtr[frameHeaderSize-1]) return err0r(LZ4F_ERROR_headerChecksum_invalid); } +#endif /* save */ dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; @@ -1210,8 +1215,10 @@ size_t LZ4F_headerSize(const void* src, size_t srcSize) return 8; /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) return err0r(LZ4F_ERROR_frameType_unknown); +#endif /* Frame Header Size */ { BYTE const FLG = ((const BYTE*)src)[4]; @@ -1493,7 +1500,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* next block is a compressed block */ dctx->tmpInTarget = nextCBlockSize + crcSize; dctx->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { + if (dstPtr==dstEnd || srcPtr==srcEnd) { nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; doAnotherStage = 0; } @@ -1554,8 +1561,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, } { U32 const readCRC = LZ4F_readLE32(crcSrc); U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != calcCRC) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#else + (void)readCRC; + (void)calcCRC; +#endif } } dctx->dStage = dstage_getBlockHeader; /* new block */ break; @@ -1594,8 +1606,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readBlockCrc != calcBlockCrc) return err0r(LZ4F_ERROR_blockChecksum_invalid); +#else + (void)readBlockCrc; + (void)calcBlockCrc; +#endif } } if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) { @@ -1723,8 +1740,13 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx, /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ { U32 const readCRC = LZ4F_readLE32(selectedIn); U32 const resultCRC = XXH32_digest(&(dctx->xxh)); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (readCRC != resultCRC) return err0r(LZ4F_ERROR_contentChecksum_invalid); +#else + (void)readCRC; + (void)resultCRC; +#endif nextSrcSizeHint = 0; LZ4F_resetDecompressionContext(dctx); doAnotherStage = 0; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 742c252..391e484 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -253,6 +253,15 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); #define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected paramaters */ #define LZ4F_HEADER_SIZE_MAX 19 +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +#define LZ4F_BLOCK_HEADER_SIZE 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +#define LZ4F_BLOCK_CHECKSUM_SIZE 4 + +/* Size in bytes of the content checksum. */ +#define LZ4F_CONTENT_CHECKSUM_SIZE 4 + /*! LZ4F_compressBegin() : * will write the frame header into dstBuffer. * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 936f739..5922ed7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -151,6 +151,21 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, return back; } +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +{ + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) + return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); +} + /* LZ4HC_countPattern() : * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ static unsigned @@ -203,6 +218,16 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) return (unsigned)(iStart - ip); } +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} + typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; @@ -228,7 +253,7 @@ LZ4HC_InsertAndGetWiderMatch ( const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const U32 ipIndex = (U32)(ip - base); - const U32 lowestMatchIndex = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const U32 lowestMatchIndex = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; const BYTE* const dictBase = hc4->dictBase; int const lookBackLength = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; @@ -287,14 +312,21 @@ LZ4HC_InsertAndGetWiderMatch ( if (chainSwap && matchLength==longest) { /* better match => select a better chain */ assert(lookBackLength==0); /* search forward only */ if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; int pos; - for (pos = 0; pos <= longest - MINMATCH; pos++) { + for (pos = 0; pos < end; pos += step) { U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); if (candidateDist > distanceToNextMatch) { distanceToNextMatch = candidateDist; matchChainPos = (U32)pos; - } } + accel = 1 << kTrigger; + } + } if (distanceToNextMatch > 1) { if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ matchIndex -= distanceToNextMatch; @@ -313,34 +345,61 @@ LZ4HC_InsertAndGetWiderMatch ( } else { repeat = rep_not; } } - if ( (repeat == rep_confirmed) - && (matchCandidateIdx >= dictLimit) ) { /* same segment only */ - const BYTE* const matchPtr = base + matchCandidateIdx; + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(dictLimit, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < dictLimit; + const BYTE* const matchPtr = (extDict ? dictBase : base) + matchCandidateIdx; if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ - size_t const forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); - const BYTE* const lowestMatchPtr = (lowPrefixPtr + LZ4_DISTANCE_MAX >= ip) ? lowPrefixPtr : ip - LZ4_DISTANCE_MAX; - size_t const backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); - size_t const currentSegmentLength = backLength + forwardPatternLength; - - if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ - && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ - matchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ - } else { - matchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ - if (lookBackLength==0) { /* no back possible */ - size_t const maxML = MIN(currentSegmentLength, srcPatternLength); - if ((size_t)longest < maxML) { - assert(base + matchIndex < ip); - if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; - assert(maxML < 2 GB); - longest = (int)maxML; - *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ - *startpos = ip; + const BYTE* const dictStart = dictBase + hc4->lowLimit; + const BYTE* const iLimit = extDict ? dictBase + dictLimit : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(lowPrefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : lowPrefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict && matchPtr - backLength == lowPrefixPtr && hc4->lowLimit < dictLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictBase + dictLimit, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; } - { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); - if (distToNextPattern > matchIndex) break; /* avoid overflow */ - matchIndex -= distToNextPattern; - } } } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(dictLimit, newMatchIndex)) { + assert(newMatchIndex >= dictLimit - 3 && newMatchIndex < dictLimit && !extDict); + matchIndex = dictLimit; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(base + matchIndex < ip); + if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + *matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */ + *startpos = ip; + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } continue; } } } } /* PA optimization */ @@ -1005,6 +1064,9 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, diff --git a/lib/lz4hc.h b/lib/lz4hc.h index cdc6d89..44e35bb 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -336,6 +336,9 @@ LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionL #ifndef LZ4_HC_SLO_098092834 #define LZ4_HC_SLO_098092834 +#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */ +#include "lz4.h" + #if defined (__cplusplus) extern "C" { #endif diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile new file mode 100644 index 0000000..6875eb6 --- /dev/null +++ b/ossfuzz/Makefile @@ -0,0 +1,74 @@ +# ########################################################################## +# LZ4 oss fuzzer - Makefile +# +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - LZ4 homepage : http://www.lz4.org +# - LZ4 source repository : https://github.com/lz4/lz4 +# ########################################################################## +# compress_fuzzer : OSS Fuzz test tool +# decompress_fuzzer : OSS Fuzz test tool +# ########################################################################## + +LZ4DIR := ../lib +LIB_FUZZING_ENGINE ?= standaloneengine.o + +DEBUGLEVEL?= 1 +DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) + +LZ4_CFLAGS = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) +LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) +LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \ + -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + +FUZZERS := \ + compress_fuzzer \ + decompress_fuzzer \ + round_trip_fuzzer \ + round_trip_stream_fuzzer \ + compress_hc_fuzzer \ + round_trip_hc_fuzzer \ + compress_frame_fuzzer \ + round_trip_frame_fuzzer \ + decompress_frame_fuzzer + +all: $(FUZZERS) + +# Include a rule to build the static library if calling this target +# directly. +$(LZ4DIR)/liblz4.a: + $(MAKE) -C $(LZ4DIR) CFLAGS="$(LZ4_CFLAGS)" liblz4.a + +%.o: %.c + $(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@ + +# Generic rule for generating fuzzers +%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a + # Compile the standalone code just in case. The OSS-Fuzz code might + # override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer" + $(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o + + # Now compile the actual fuzzer. + $(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT) + +%_fuzzer_clean: + $(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o + +.PHONY: clean +clean: compress_fuzzer_clean decompress_fuzzer_clean + $(MAKE) -C $(LZ4DIR) clean diff --git a/ossfuzz/compress_frame_fuzzer.c b/ossfuzz/compress_frame_fuzzer.c new file mode 100644 index 0000000..75c609f --- /dev/null +++ b/ossfuzz/compress_frame_fuzzer.c @@ -0,0 +1,42 @@ +/** + * This fuzz target attempts to compress the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4frame.h" +#include "lz4_helpers.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed); + size_t const compressBound = LZ4F_compressFrameBound(size, &prefs); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* If compression succeeds it must round trip correctly. */ + size_t const dstSize = + LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); + if (!LZ4F_isError(dstSize)) { + size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/compress_fuzzer.c b/ossfuzz/compress_fuzzer.c new file mode 100644 index 0000000..7021624 --- /dev/null +++ b/ossfuzz/compress_fuzzer.c @@ -0,0 +1,51 @@ +/** + * This fuzz target attempts to compress the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size)); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* If compression succeeds it must round trip correctly. */ + { + int const dstSize = LZ4_compress_default((const char*)data, dst, + size, dstCapacity); + if (dstSize > 0) { + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + } + } + + if (dstCapacity > 0) { + /* Compression succeeds and must round trip correctly. */ + int compressedSize = size; + int const dstSize = LZ4_compress_destSize((const char*)data, dst, + &compressedSize, dstCapacity); + FUZZ_ASSERT(dstSize > 0); + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!"); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/compress_hc_fuzzer.c b/ossfuzz/compress_hc_fuzzer.c new file mode 100644 index 0000000..4841367 --- /dev/null +++ b/ossfuzz/compress_hc_fuzzer.c @@ -0,0 +1,57 @@ +/** + * This fuzz target attempts to compress the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4hc.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size)); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* If compression succeeds it must round trip correctly. */ + { + int const dstSize = LZ4_compress_HC((const char*)data, dst, size, + dstCapacity, level); + if (dstSize > 0) { + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + } + } + + if (dstCapacity > 0) { + /* Compression succeeds and must round trip correctly. */ + void* state = malloc(LZ4_sizeofStateHC()); + FUZZ_ASSERT(state); + int compressedSize = size; + int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data, + dst, &compressedSize, + dstCapacity, level); + FUZZ_ASSERT(dstSize > 0); + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!"); + free(state); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/decompress_frame_fuzzer.c b/ossfuzz/decompress_frame_fuzzer.c new file mode 100644 index 0000000..bda25b0 --- /dev/null +++ b/ossfuzz/decompress_frame_fuzzer.c @@ -0,0 +1,67 @@ +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#include "lz4_helpers.h" + +static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* opts) +{ + LZ4F_resetDecompressionContext(dctx); + if (dictSize == 0) + LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts); + else + LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize, + dict, dictSize, opts); +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size); + size_t const largeDictSize = 64 * 1024; + size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize); + char* const dst = (char*)malloc(dstCapacity); + char* const dict = (char*)malloc(dictSize); + LZ4F_decompressOptions_t opts; + LZ4F_dctx* dctx; + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + + FUZZ_ASSERT(dctx); + FUZZ_ASSERT(dst); + FUZZ_ASSERT(dict); + + /* Prepare the dictionary. The data doesn't matter for decompression. */ + memset(dict, 0, dictSize); + + + /* Decompress using multiple configurations. */ + memset(&opts, 0, sizeof(opts)); + opts.stableDst = 0; + decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); + opts.stableDst = 1; + decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); + opts.stableDst = 0; + decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); + opts.stableDst = 1; + decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); + + LZ4F_freeDecompressionContext(dctx); + free(dst); + free(dict); + + return 0; +} diff --git a/ossfuzz/decompress_fuzzer.c b/ossfuzz/decompress_fuzzer.c new file mode 100644 index 0000000..0267c93 --- /dev/null +++ b/ossfuzz/decompress_fuzzer.c @@ -0,0 +1,58 @@ +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size); + size_t const smallDictSize = size + 1; + size_t const largeDictSize = 64 * 1024 - 1; + size_t const dictSize = MAX(smallDictSize, largeDictSize); + char* const dst = (char*)malloc(dstCapacity); + char* const dict = (char*)malloc(dictSize + size); + char* const largeDict = dict; + char* const dataAfterDict = dict + dictSize; + char* const smallDict = dataAfterDict - smallDictSize; + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(dict); + + /* Prepare the dictionary. The data doesn't matter for decompression. */ + memset(dict, 0, dictSize); + memcpy(dataAfterDict, data, size); + + /* Decompress using each possible dictionary configuration. */ + /* No dictionary. */ + LZ4_decompress_safe_usingDict((char const*)data, dst, size, + dstCapacity, NULL, 0); + /* Small external dictonary. */ + LZ4_decompress_safe_usingDict((char const*)data, dst, size, + dstCapacity, smallDict, smallDictSize); + /* Large external dictionary. */ + LZ4_decompress_safe_usingDict((char const*)data, dst, size, + dstCapacity, largeDict, largeDictSize); + /* Small prefix. */ + LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size, + dstCapacity, smallDict, smallDictSize); + /* Large prefix. */ + LZ4_decompress_safe_usingDict((char const*)data, dst, size, + dstCapacity, largeDict, largeDictSize); + /* Partial decompression. */ + LZ4_decompress_safe_partial((char const*)data, dst, size, + dstCapacity, dstCapacity); + free(dst); + free(dict); + + return 0; +} diff --git a/ossfuzz/fuzz.h b/ossfuzz/fuzz.h new file mode 100644 index 0000000..eefac63 --- /dev/null +++ b/ossfuzz/fuzz.h @@ -0,0 +1,48 @@ +/** + * Fuzz target interface. + * Fuzz targets have some common parameters passed as macros during compilation. + * Check the documentation for each individual fuzzer for more parameters. + * + * @param FUZZ_RNG_SEED_SIZE: + * The number of bytes of the source to look at when constructing a seed + * for the deterministic RNG. These bytes are discarded before passing + * the data to lz4 functions. Every fuzzer initializes the RNG exactly + * once before doing anything else, even if it is unused. + * Default: 4. + * @param LZ4_DEBUG: + * This is a parameter for the lz4 library. Defining `LZ4_DEBUG=1` + * enables assert() statements in the lz4 library. Higher levels enable + * logging, so aren't recommended. Defining `LZ4_DEBUG=1` is + * recommended. + * @param LZ4_FORCE_MEMORY_ACCESS: + * This flag controls how the zstd library accesses unaligned memory. + * It can be undefined, or 0 through 2. If it is undefined, it selects + * the method to use based on the compiler. If testing with UBSAN set + * MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method. + * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + * This is the canonical flag to enable deterministic builds for fuzzing. + * Changes to zstd for fuzzing are gated behind this define. + * It is recommended to define this when building zstd for fuzzing. + */ + +#ifndef FUZZ_H +#define FUZZ_H + +#ifndef FUZZ_RNG_SEED_SIZE +# define FUZZ_RNG_SEED_SIZE 4 +#endif + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ossfuzz/fuzz_helpers.h b/ossfuzz/fuzz_helpers.h new file mode 100644 index 0000000..c4a8645 --- /dev/null +++ b/ossfuzz/fuzz_helpers.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * Helper functions for fuzzing. + */ + +#ifndef FUZZ_HELPERS_H +#define FUZZ_HELPERS_H + +#include "fuzz.h" +#include "xxhash.h" +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define LZ4_COMMONDEFS_ONLY +#ifndef LZ4_SRC_INCLUDED +#include "lz4.c" /* LZ4_count, constants, mem */ +#endif + +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + +#define FUZZ_QUOTE_IMPL(str) #str +#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str) + +/** + * Asserts for fuzzing that are always enabled. + */ +#define FUZZ_ASSERT_MSG(cond, msg) \ + ((cond) ? (void)0 \ + : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \ + __LINE__, FUZZ_QUOTE(cond), (msg)), \ + abort())) +#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); + +#if defined(__GNUC__) +#define FUZZ_STATIC static __inline __attribute__((unused)) +#elif defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#define FUZZ_STATIC static inline +#elif defined(_MSC_VER) +#define FUZZ_STATIC static __inline +#else +#define FUZZ_STATIC static +#endif + +/** + * Deterministically constructs a seed based on the fuzz input. + * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. + */ +FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { + uint8_t const *data = *src; + size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); + *size -= toHash; + *src += toHash; + return XXH32(data, toHash, 0); +} + +#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) + +FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { + static const uint32_t prime1 = 2654435761U; + static const uint32_t prime2 = 2246822519U; + uint32_t rand32 = *state; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZZ_rotl32(rand32, 13); + *state = rand32; + return rand32 >> 5; +} + +/* Returns a random numer in the range [min, max]. */ +FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { + uint32_t random = FUZZ_rand(state); + return min + (random % (max - min + 1)); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ossfuzz/lz4_helpers.c b/ossfuzz/lz4_helpers.c new file mode 100644 index 0000000..9471630 --- /dev/null +++ b/ossfuzz/lz4_helpers.c @@ -0,0 +1,51 @@ +#include "fuzz_helpers.h" +#include "lz4_helpers.h" +#include "lz4hc.h" + +LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed) +{ + LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO; + info.blockSizeID = FUZZ_rand32(seed, LZ4F_max64KB - 1, LZ4F_max4MB); + if (info.blockSizeID < LZ4F_max64KB) { + info.blockSizeID = LZ4F_default; + } + info.blockMode = FUZZ_rand32(seed, LZ4F_blockLinked, LZ4F_blockIndependent); + info.contentChecksumFlag = FUZZ_rand32(seed, LZ4F_noContentChecksum, + LZ4F_contentChecksumEnabled); + info.blockChecksumFlag = FUZZ_rand32(seed, LZ4F_noBlockChecksum, + LZ4F_blockChecksumEnabled); + return info; +} + +LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed) +{ + LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; + prefs.frameInfo = FUZZ_randomFrameInfo(seed); + prefs.compressionLevel = FUZZ_rand32(seed, 0, LZ4HC_CLEVEL_MAX + 3) - 3; + prefs.autoFlush = FUZZ_rand32(seed, 0, 1); + prefs.favorDecSpeed = FUZZ_rand32(seed, 0, 1); + return prefs; +} + +size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, + const void* src, const size_t srcSize) +{ + LZ4F_decompressOptions_t opts; + memset(&opts, 0, sizeof(opts)); + opts.stableDst = 1; + LZ4F_dctx* dctx; + LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); + FUZZ_ASSERT(dctx); + + size_t dstSize = dstCapacity; + size_t srcConsumed = srcSize; + size_t const rc = + LZ4F_decompress(dctx, dst, &dstSize, src, &srcConsumed, &opts); + FUZZ_ASSERT(!LZ4F_isError(rc)); + FUZZ_ASSERT(rc == 0); + FUZZ_ASSERT(srcConsumed == srcSize); + + LZ4F_freeDecompressionContext(dctx); + + return dstSize; +} diff --git a/ossfuzz/lz4_helpers.h b/ossfuzz/lz4_helpers.h new file mode 100644 index 0000000..c99fb01 --- /dev/null +++ b/ossfuzz/lz4_helpers.h @@ -0,0 +1,13 @@ +#ifndef LZ4_HELPERS +#define LZ4_HELPERS + +#include "lz4frame.h" + +LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed); + +LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed); + +size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, + const void* src, const size_t srcSize); + +#endif /* LZ4_HELPERS */ diff --git a/ossfuzz/ossfuzz.sh b/ossfuzz/ossfuzz.sh new file mode 100755 index 0000000..9782286 --- /dev/null +++ b/ossfuzz/ossfuzz.sh @@ -0,0 +1,23 @@ +#!/bin/bash -eu + +# This script is called by the oss-fuzz main project when compiling the fuzz +# targets. This script is regression tested by travisoss.sh. + +# Save off the current folder as the build root. +export BUILD_ROOT=$PWD + +echo "CC: $CC" +echo "CXX: $CXX" +echo "LIB_FUZZING_ENGINE: $LIB_FUZZING_ENGINE" +echo "CFLAGS: $CFLAGS" +echo "CXXFLAGS: $CXXFLAGS" +echo "OUT: $OUT" + +export MAKEFLAGS+="-j$(nproc)" + +pushd ossfuzz +make V=1 all +popd + +# Copy the fuzzers to the target directory. +cp -v ossfuzz/*_fuzzer $OUT/ diff --git a/ossfuzz/round_trip_frame_fuzzer.c b/ossfuzz/round_trip_frame_fuzzer.c new file mode 100644 index 0000000..1eea90c --- /dev/null +++ b/ossfuzz/round_trip_frame_fuzzer.c @@ -0,0 +1,39 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4frame.h" +#include "lz4_helpers.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed); + size_t const dstCapacity = LZ4F_compressFrameBound(size, &prefs); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + size_t const dstSize = + LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); + FUZZ_ASSERT(!LZ4F_isError(dstSize)); + size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/round_trip_fuzzer.c b/ossfuzz/round_trip_fuzzer.c new file mode 100644 index 0000000..3a66e80 --- /dev/null +++ b/ossfuzz/round_trip_fuzzer.c @@ -0,0 +1,50 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = LZ4_compressBound(size); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + int const dstSize = LZ4_compress_default((const char*)data, dst, + size, dstCapacity); + FUZZ_ASSERT(dstSize > 0); + + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + + /* Partial decompression must succeed. */ + { + size_t const partialCapacity = FUZZ_rand32(&seed, 0, size); + char* const partial = (char*)malloc(partialCapacity); + FUZZ_ASSERT(partial); + int const partialSize = LZ4_decompress_safe_partial( + dst, partial, dstSize, partialCapacity, partialCapacity); + FUZZ_ASSERT(partialSize >= 0); + FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); + FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); + free(partial); + } + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/round_trip_hc_fuzzer.c b/ossfuzz/round_trip_hc_fuzzer.c new file mode 100644 index 0000000..325cdf0 --- /dev/null +++ b/ossfuzz/round_trip_hc_fuzzer.c @@ -0,0 +1,39 @@ +/** + * This fuzz target performs a lz4 round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#include "lz4.h" +#include "lz4hc.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + size_t const dstCapacity = LZ4_compressBound(size); + char* const dst = (char*)malloc(dstCapacity); + char* const rt = (char*)malloc(size); + int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + + FUZZ_ASSERT(dst); + FUZZ_ASSERT(rt); + + /* Compression must succeed and round trip correctly. */ + int const dstSize = LZ4_compress_HC((const char*)data, dst, size, + dstCapacity, level); + FUZZ_ASSERT(dstSize > 0); + + int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); + FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size"); + FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); + + free(dst); + free(rt); + + return 0; +} diff --git a/ossfuzz/round_trip_stream_fuzzer.c b/ossfuzz/round_trip_stream_fuzzer.c new file mode 100644 index 0000000..abfcd2d --- /dev/null +++ b/ossfuzz/round_trip_stream_fuzzer.c @@ -0,0 +1,302 @@ +/** + * This fuzz target performs a lz4 streaming round-trip test + * (compress & decompress), compares the result with the original, and calls + * abort() on corruption. + */ + +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "fuzz_helpers.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" + +typedef struct { + char const* buf; + size_t size; + size_t pos; +} const_cursor_t; + +typedef struct { + char* buf; + size_t size; + size_t pos; +} cursor_t; + +typedef struct { + LZ4_stream_t* cstream; + LZ4_streamHC_t* cstreamHC; + LZ4_streamDecode_t* dstream; + const_cursor_t data; + cursor_t compressed; + cursor_t roundTrip; + uint32_t seed; + int level; +} state_t; + +cursor_t cursor_create(size_t size) +{ + cursor_t cursor; + cursor.buf = (char*)malloc(size); + cursor.size = size; + cursor.pos = 0; + FUZZ_ASSERT(cursor.buf); + return cursor; +} + +typedef void (*round_trip_t)(state_t* state); + +void cursor_free(cursor_t cursor) +{ + free(cursor.buf); +} + +state_t state_create(char const* data, size_t size, uint32_t seed) +{ + state_t state; + + state.seed = seed; + + state.data.buf = (char const*)data; + state.data.size = size; + state.data.pos = 0; + + /* Extra margin because we are streaming. */ + state.compressed = cursor_create(1024 + 2 * LZ4_compressBound(size)); + state.roundTrip = cursor_create(size); + + state.cstream = LZ4_createStream(); + FUZZ_ASSERT(state.cstream); + state.cstreamHC = LZ4_createStreamHC(); + FUZZ_ASSERT(state.cstream); + state.dstream = LZ4_createStreamDecode(); + FUZZ_ASSERT(state.dstream); + + return state; +} + +void state_free(state_t state) +{ + cursor_free(state.compressed); + cursor_free(state.roundTrip); + LZ4_freeStream(state.cstream); + LZ4_freeStreamHC(state.cstreamHC); + LZ4_freeStreamDecode(state.dstream); +} + +static void state_reset(state_t* state, uint32_t seed) +{ + state->level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); + LZ4_resetStream_fast(state->cstream); + LZ4_resetStreamHC_fast(state->cstreamHC, state->level); + LZ4_setStreamDecode(state->dstream, NULL, 0); + state->data.pos = 0; + state->compressed.pos = 0; + state->roundTrip.pos = 0; + state->seed = seed; +} + +static void state_decompress(state_t* state, char const* src, int srcSize) +{ + char* dst = state->roundTrip.buf + state->roundTrip.pos; + int const dstCapacity = state->roundTrip.size - state->roundTrip.pos; + int const dSize = LZ4_decompress_safe_continue(state->dstream, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(dSize >= 0); + state->roundTrip.pos += dSize; +} + +static void state_checkRoundTrip(state_t const* state) +{ + char const* data = state->data.buf; + size_t const size = state->data.size; + FUZZ_ASSERT_MSG(size == state->roundTrip.pos, "Incorrect size!"); + FUZZ_ASSERT_MSG(!memcmp(data, state->roundTrip.buf, size), "Corruption!"); +} + +/** + * Picks a dictionary size and trims the dictionary off of the data. + * We copy the dictionary to the roundTrip so our validation passes. + */ +static size_t state_trimDict(state_t* state) +{ + /* 64 KB is the max dict size, allow slightly beyond that to test trim. */ + uint32_t maxDictSize = MIN(70 * 1024, state->data.size); + size_t const dictSize = FUZZ_rand32(&state->seed, 0, maxDictSize); + DEBUGLOG(2, "dictSize = %zu", dictSize); + FUZZ_ASSERT(state->data.pos == 0); + FUZZ_ASSERT(state->roundTrip.pos == 0); + memcpy(state->roundTrip.buf, state->data.buf, dictSize); + state->data.pos += dictSize; + state->roundTrip.pos += dictSize; + return dictSize; +} + +static void state_prefixRoundTrip(state_t* state) +{ + while (state->data.pos != state->data.size) { + char const* src = state->data.buf + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst, + srcSize, dstCapacity, 0); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } +} + +static void state_extDictRoundTrip(state_t* state) +{ + int i = 0; + cursor_t data2 = cursor_create(state->data.size); + memcpy(data2.buf, state->data.buf, state->data.size); + while (state->data.pos != state->data.size) { + char const* data = (i++ & 1) ? state->data.buf : data2.buf; + char const* src = data + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst, + srcSize, dstCapacity, 0); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } + cursor_free(data2); +} + +static void state_randomRoundTrip(state_t* state, round_trip_t rt0, + round_trip_t rt1) +{ + if (FUZZ_rand32(&state->seed, 0, 1)) { + rt0(state); + } else { + rt1(state); + } +} + +static void state_loadDictRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_loadDict(state->cstream, dict, dictSize); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip); +} + +static void state_attachDictRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_stream_t* dictStream = LZ4_createStream(); + LZ4_loadDict(dictStream, dict, dictSize); + LZ4_attach_dictionary(state->cstream, dictStream); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip); + LZ4_freeStream(dictStream); +} + +static void state_prefixHCRoundTrip(state_t* state) +{ + while (state->data.pos != state->data.size) { + char const* src = state->data.buf + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(cSize > 0); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } +} + +static void state_extDictHCRoundTrip(state_t* state) +{ + int i = 0; + cursor_t data2 = cursor_create(state->data.size); + DEBUGLOG(2, "extDictHC"); + memcpy(data2.buf, state->data.buf, state->data.size); + while (state->data.pos != state->data.size) { + char const* data = (i++ & 1) ? state->data.buf : data2.buf; + char const* src = data + state->data.pos; + char* dst = state->compressed.buf + state->compressed.pos; + int const srcRemaining = state->data.size - state->data.pos; + int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining); + int const dstCapacity = state->compressed.size - state->compressed.pos; + int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst, + srcSize, dstCapacity); + FUZZ_ASSERT(cSize > 0); + DEBUGLOG(2, "srcSize = %d", srcSize); + state->data.pos += srcSize; + state->compressed.pos += cSize; + state_decompress(state, dst, cSize); + } + cursor_free(data2); +} + +static void state_loadDictHCRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_loadDictHC(state->cstreamHC, dict, dictSize); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixHCRoundTrip, + state_extDictHCRoundTrip); +} + +static void state_attachDictHCRoundTrip(state_t* state) +{ + char const* dict = state->data.buf; + size_t const dictSize = state_trimDict(state); + LZ4_streamHC_t* dictStream = LZ4_createStreamHC(); + LZ4_setCompressionLevel(dictStream, state->level); + LZ4_loadDictHC(dictStream, dict, dictSize); + LZ4_attach_HC_dictionary(state->cstreamHC, dictStream); + LZ4_setStreamDecode(state->dstream, dict, dictSize); + state_randomRoundTrip(state, state_prefixHCRoundTrip, + state_extDictHCRoundTrip); + LZ4_freeStreamHC(dictStream); +} + +round_trip_t roundTrips[] = { + &state_prefixRoundTrip, + &state_extDictRoundTrip, + &state_loadDictRoundTrip, + &state_attachDictRoundTrip, + &state_prefixHCRoundTrip, + &state_extDictHCRoundTrip, + &state_loadDictHCRoundTrip, + &state_attachDictHCRoundTrip, +}; + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint32_t seed = FUZZ_seed(&data, &size); + state_t state = state_create((char const*)data, size, seed); + const int n = sizeof(roundTrips) / sizeof(round_trip_t); + int i; + + for (i = 0; i < n; ++i) { + DEBUGLOG(2, "Round trip %d", i); + state_reset(&state, seed); + roundTrips[i](&state); + state_checkRoundTrip(&state); + } + + state_free(state); + + return 0; +} diff --git a/ossfuzz/standaloneengine.c b/ossfuzz/standaloneengine.c new file mode 100644 index 0000000..6afeffd --- /dev/null +++ b/ossfuzz/standaloneengine.c @@ -0,0 +1,74 @@ +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#include "fuzz.h" + +/** + * Main procedure for standalone fuzzing engine. + * + * Reads filenames from the argument array. For each filename, read the file + * into memory and then call the fuzzing interface with the data. + */ +int main(int argc, char **argv) +{ + int ii; + for(ii = 1; ii < argc; ii++) + { + FILE *infile; + printf("[%s] ", argv[ii]); + + /* Try and open the file. */ + infile = fopen(argv[ii], "rb"); + if(infile) + { + uint8_t *buffer = NULL; + size_t buffer_len; + + printf("Opened.. "); + + /* Get the length of the file. */ + fseek(infile, 0L, SEEK_END); + buffer_len = ftell(infile); + + /* Reset the file indicator to the beginning of the file. */ + fseek(infile, 0L, SEEK_SET); + + /* Allocate a buffer for the file contents. */ + buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t)); + if(buffer) + { + /* Read all the text from the file into the buffer. */ + fread(buffer, sizeof(uint8_t), buffer_len, infile); + printf("Read %zu bytes, fuzzing.. ", buffer_len); + + /* Call the fuzzer with the data. */ + LLVMFuzzerTestOneInput(buffer, buffer_len); + + printf("complete !!"); + + /* Free the buffer as it's no longer needed. */ + free(buffer); + buffer = NULL; + } + else + { + fprintf(stderr, + "[%s] Failed to allocate %zu bytes \n", + argv[ii], + buffer_len); + } + + /* Close the file as it's no longer needed. */ + fclose(infile); + infile = NULL; + } + else + { + /* Failed to open the file. Maybe wrong name or wrong permissions? */ + fprintf(stderr, "[%s] Open failed. \n", argv[ii]); + } + + printf("\n"); + } +} diff --git a/ossfuzz/travisoss.sh b/ossfuzz/travisoss.sh new file mode 100755 index 0000000..5ea884c --- /dev/null +++ b/ossfuzz/travisoss.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +# Clone the oss-fuzz repository +git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz + +if [[ ! -d /tmp/ossfuzz/projects/lz4 ]] +then + echo "Could not find the lz4 project in ossfuzz" + exit 1 +fi + +# Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis. +sed -i "s@https://github.com/lz4/lz4.git@-b $TRAVIS_BRANCH https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile + +# Try and build the fuzzers +pushd /tmp/ossfuzz +python infra/helper.py build_image --pull lz4 +python infra/helper.py build_fuzzers lz4 +popd diff --git a/programs/lz4.1 b/programs/lz4.1 index ad0c12c..d758ed5 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -1,5 +1,5 @@ . -.TH "LZ4" "1" "April 2019" "lz4 1.9.1" "User Commands" +.TH "LZ4" "1" "July 2019" "lz4 1.9.2" "User Commands" . .SH "NAME" \fBlz4\fR \- lz4, unlz4, lz4cat \- Compress or decompress \.lz4 files @@ -120,6 +120,10 @@ Compression level, with # being any value from 1 to 12\. Higher values trade com Switch to ultra\-fast compression levels\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. If \fB=#\fR is not present, it defaults to \fB1\fR\. This setting overrides compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\. . .TP +\fB\-\-best\fR +Set highest compression level\. Same as -12\. +. +.TP \fB\-\-favor\-decSpeed\fR Generate compressed data optimized for decompression speed\. Compressed data will be larger as a consequence (typically by ~0\.5%), while decompression speed will be improved by 5\-20%, depending on use cases\. This option only works in combination with very high compression levels (>=10)\. . diff --git a/programs/lz4.1.md b/programs/lz4.1.md index 8874467..56c0053 100644 --- a/programs/lz4.1.md +++ b/programs/lz4.1.md @@ -135,6 +135,9 @@ only the latest one will be applied. This setting overrides compression level if one was set previously. Similarly, if a compression level is set after `--fast`, it overrides it. +* `--best`: + Set highest compression level. Same as -12. + * `--favor-decSpeed`: Generate compressed data optimized for decompression speed. Compressed data will be larger as a consequence (typically by ~0.5%), diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 39ff1ea..5da7654 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -67,6 +67,7 @@ static int g_lz4c_legacy_commands = 0; /*-************************************ * Macros ***************************************/ +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ @@ -141,10 +142,11 @@ static int usage_advanced(const char* exeName) DISPLAY( " -BX : enable block checksum (default:disabled) \n"); DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--list : lists information about .lz4 files. Useful if compressed with --content-size flag.\n"); + DISPLAY( "--list FILE : lists information about .lz4 files (useful for files compressed with --content-size flag)\n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); DISPLAY( "--favor-decSpeed: compressed files decompress faster, but are less compressed \n"); DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %i)\n", 1); + DISPLAY( "--best : same as -%d\n", LZ4HC_CLEVEL_MAX); DISPLAY( "Benchmark arguments : \n"); DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); @@ -390,7 +392,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } - if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } + if (!strcmp(argument, "--version")) { DISPLAYOUT(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; } if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; } /* keep source file (default) */ if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; } @@ -413,6 +415,9 @@ int main(int argc, const char** argv) } continue; } + + /* For gzip(1) compatibility */ + if (!strcmp(argument, "--best")) { cLevel=LZ4HC_CLEVEL_MAX; continue; } } while (argument[1]!=0) { @@ -437,7 +442,7 @@ int main(int argc, const char** argv) switch(argument[0]) { /* Display help */ - case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */ + case 'V': DISPLAYOUT(WELCOME_MESSAGE); goto _cleanup; /* Version */ case 'h': usage_advanced(exeName); goto _cleanup; case 'H': usage_longhelp(exeName); goto _cleanup; @@ -648,13 +653,19 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } - /* if input==stdin and no output defined, stdout becomes default output */ - if (!strcmp(input_filename, stdinmark) && !output_filename) - output_filename = stdoutmark; + if (!strcmp(input_filename, stdinmark)) { + /* if input==stdin and no output defined, stdout becomes default output */ + if (!output_filename) output_filename = stdoutmark; + } + else{ + if (!recursive && !UTIL_isRegFile(input_filename)) { + DISPLAYLEVEL(1, "%s: is not a regular file \n", input_filename); + exit(1); + } + } /* No output filename ==> try to select one automatically (when possible) */ while ((!output_filename) && (multiple_inputs==0)) { - if (!IS_CONSOLE(stdout)) { /* Default to stdout whenever stdout is not the console. * Note : this policy may change in the future, therefore don't rely on it ! @@ -693,7 +704,19 @@ int main(int argc, const char** argv) break; } - if (multiple_inputs==0 && mode != om_list) assert(output_filename); + if (mode == om_list){ + /* Exit if trying to read from stdin as this isn't supported in this mode */ + if(!strcmp(input_filename, stdinmark)){ + DISPLAYLEVEL(1, "refusing to read from standard input in --list mode\n"); + exit(1); + } + if(!multiple_inputs){ + inFileNames[ifnIdx++] = input_filename; + } + } + else{ + if (multiple_inputs==0) assert(output_filename); + } /* when multiple_inputs==1, output_filename may simply be useless, * however, output_filename must be !NULL for next strcmp() tests */ if (!output_filename) output_filename = "*\\dummy^!//"; @@ -723,11 +746,7 @@ int main(int argc, const char** argv) operationResult = DEFAULT_DECOMPRESSOR(prefs, input_filename, output_filename); } } else if (mode == om_list){ - if(!multiple_inputs){ - inFileNames[ifnIdx++] = input_filename; - } operationResult = LZ4IO_displayCompressedFilesInfo(inFileNames, ifnIdx); - inFileNames=NULL; } else { /* compression is default action */ if (legacy_format) { DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); diff --git a/programs/lz4io.c b/programs/lz4io.c index 960c451..d818535 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -102,6 +102,7 @@ static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result } } static const clock_t refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_time = 0; +#define LZ4IO_STATIC_ASSERT(c) { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ /************************************** @@ -218,8 +219,8 @@ size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid) static const unsigned minBlockSizeID = 4; static const unsigned maxBlockSizeID = 7; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - prefs->blockSizeId = bsid; - prefs->blockSize = blockSizeTable[prefs->blockSizeId-minBlockSizeID]; + prefs->blockSizeId = (int)bsid; + prefs->blockSize = blockSizeTable[(unsigned)prefs->blockSizeId-minBlockSizeID]; return prefs->blockSize; } @@ -236,7 +237,7 @@ size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize) while (blockSize >>= 2) bsid++; if (bsid < 7) bsid = 7; - prefs->blockSizeId = bsid-3; + prefs->blockSizeId = (int)(bsid-3); return prefs->blockSize; } @@ -416,7 +417,7 @@ int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_ /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(outBuffSize + 4); + out_buff = (char*)malloc((size_t)outBuffSize + 4); if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); @@ -897,7 +898,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F unsigned storedSkips = 0; /* Allocate Memory */ - char* const in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + char* const in_buff = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE)); char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE); if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); @@ -921,11 +922,11 @@ static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, F if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); } /* Decode Block */ - { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + { int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE); if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); - streamSize += decodeSize; + streamSize += (unsigned long long)decodeSize; /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, decodeSize, storedSkips); /* success or die */ + storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, (size_t)decodeSize, storedSkips); /* success or die */ } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); @@ -1278,131 +1279,331 @@ int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, /* ********************** LZ4 --list command *********************** */ /* ********************************************************************* */ +typedef enum +{ + lz4Frame = 0, + legacyFrame, + skippableFrame +} LZ4IO_frameType_t; + typedef struct { - LZ4F_frameInfo_t frameInfo; - const char* fileName; - unsigned long long fileSize; -} LZ4IO_cFileInfo_t; + LZ4F_frameInfo_t lz4FrameInfo; + LZ4IO_frameType_t frameType; +} LZ4IO_frameInfo_t; -#define LZ4IO_INIT_CFILEINFO { LZ4F_INIT_FRAMEINFO, NULL, 0ULL } +#define LZ4IO_INIT_FRAMEINFO { LZ4F_INIT_FRAMEINFO, lz4Frame } +typedef struct { + const char* fileName; + unsigned long long fileSize; + unsigned long long frameCount; + LZ4IO_frameInfo_t frameSummary; + unsigned short eqFrameTypes; + unsigned short eqBlockTypes; + unsigned short allContentSize; +} LZ4IO_cFileInfo_t; -typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; +#define LZ4IO_INIT_CFILEINFO { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 } -/* This function is limited, - * it only works fine for a file consisting of a single valid frame using LZ4 Frame specification. - * It will not look at content beyond first frame header. - * It's also unable to parse legacy frames, nor skippable ones. - * - * Things to improve : - * - check the entire file for additional content after first frame - * + combine results from multiple frames, give total - * - Optional : - * + report nb of blocks, hence max. possible decompressed size (when not reported in header) - */ -static LZ4IO_infoResult -LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) -{ - LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ +typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult; - if (!UTIL_isRegFile(input_filename)) return LZ4IO_not_a_file; - cfinfo->fileSize = UTIL_getFileSize(input_filename); +static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "SkippableFrame" }; - /* Get filename without path prefix */ - { const char* b = strrchr(input_filename, '/'); - if (!b) { - b = strrchr(input_filename, '\\'); +/* Read block headers and skip block data + Return total blocks size for this frame including block headers, + block checksums and content checksums. + returns 0 in case it can't succesfully skip block data. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long LZ4IO_skipBlocksData(FILE* finput, + const LZ4F_blockChecksum_t blockChecksumFlag, + const LZ4F_contentChecksum_t contentChecksumFlag) { + unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + for (;;) { + if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; + return 0; } - if (b && b != input_filename) { - b++; - } else { - b = input_filename; + totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE; + { + const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU; + const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE); + if (nextCBlockSize == 0) { + /* Reached EndMark */ + if (contentChecksumFlag) { + /* Skip content checksum */ + if (UTIL_fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0) { + return 0; + } + totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE; + } + break; + } + totalBlocksSize += nextBlock; + /* skip to the next block */ + if (UTIL_fseek(finput, nextBlock, SEEK_CUR) != 0) { + return 0; + } } - cfinfo->fileName = b; } + return totalBlocksSize; +} - /* Read file and extract header */ - { size_t const hSize = LZ4F_HEADER_SIZE_MAX; - size_t readSize=0; - - void* const buffer = malloc(hSize); - if (!buffer) EXM_THROW(21, "Allocation error : not enough memory"); - - { FILE* const finput = LZ4IO_openSrcFile(input_filename); - if (finput) { - readSize = fread(buffer, 1, hSize, finput); - fclose(finput); - } } - - if (readSize > 0) { - LZ4F_dctx* dctx; - if (!LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) { - if (!LZ4F_isError(LZ4F_getFrameInfo(dctx, &cfinfo->frameInfo, buffer, &readSize))) { - result = LZ4IO_LZ4F_OK; - } } - LZ4F_freeDecompressionContext(dctx); +/* For legacy frames only. + Read block headers and skip block data. + Return total blocks size for this frame including block headers. + or 0 in case it can't succesfully skip block data. + This works as long as legacy block header size = magic number size. + Assumes SEEK_CUR after frame header. + */ +static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput) { + unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE]; + unsigned long long totalBlocksSize = 0; + LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE); + for (;;) { + if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) { + if (feof(finput)) return totalBlocksSize; + return 0; + } + { const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo); + if ( nextCBlockSize == LEGACY_MAGICNUMBER || + nextCBlockSize == LZ4IO_MAGICNUMBER || + LZ4IO_isSkippableMagicNumber(nextCBlockSize)) { + /* Rewind back. we want cursor at the begining of next frame.*/ + if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) { + return 0; + } + break; + } + totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize; + /* skip to the next block */ + if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) { + return 0; + } } - - /* clean */ - free(buffer); } - - return result; + return totalBlocksSize; } - /* buffer : must be a valid memory area of at least 4 bytes */ -const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) -{ +const char* LZ4IO_blockTypeID(int sizeID, int blockMode, char* buffer) { buffer[0] = 'B'; - assert(sizeID >= 4); assert(sizeID <=7); + assert(sizeID >= 4); assert(sizeID <= 7); buffer[1] = (char)(sizeID + '0'); buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D'; buffer[3] = 0; return buffer; } +/* buffer : must be valid memory area of at least 10 bytes */ +static const char* LZ4IO_toHuman(long double size, char *buf) { + const char units[] = {"\0KMGTPEZY"}; + size_t i = 0; + for (; size >= 1024; i++) size /= 1024; + sprintf(buf, "%.2Lf%c", size, units[i]); + return buf; +} -int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) +/* Get filename without path prefix */ +static const char* LZ4IO_baseName(const char* input_filename) { + const char* b = strrchr(input_filename, '/'); + if (!b) b = strrchr(input_filename, '\\'); + if (!b) return input_filename; + return b ? b + 1 : b; +} + +/* Report frame/s information in verbose mode. + * Will populate file info with fileName and frameSummary where applicable. + * - TODO : + * + report nb of blocks, hence max. possible decompressed size (when not reported in header) + */ +static LZ4IO_infoResult +LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename) { - int result = 0; - size_t idx; + LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */ + unsigned char buffer[LZ4F_HEADER_SIZE_MAX]; + FILE* const finput = LZ4IO_openSrcFile(input_filename); + cfinfo->fileSize = UTIL_getFileSize(input_filename); + + while (!feof(finput)) { + LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO; + unsigned magicNumber; + /* Get MagicNumber */ + size_t nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes == 0) { break; } /* EOF */ + result = LZ4IO_format_not_known; /* default result (error) */ + if (nbReadBytes != MAGICNUMBER_SIZE) + EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LZ4IO_readLE32(buffer); /* Little Endian format */ + if (LZ4IO_isSkippableMagicNumber(magicNumber)) + magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ + + switch (magicNumber) { + case LZ4IO_MAGICNUMBER: + if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0; + /* Get frame info */ + { const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput); + if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename); + } + { size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN); + if (!LZ4F_isError(hSize)) { + if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) { + /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/ + const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput); + if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename); + } + /* Create decompression context */ + { LZ4F_dctx* dctx; + unsigned isError = LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)); + if (!isError) { + isError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize)); + LZ4F_freeDecompressionContext(dctx); + if (!isError) { + if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID || + cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode) + && cfinfo->frameCount != 0) + cfinfo->eqBlockTypes = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput, + frameInfo.lz4FrameInfo.blockChecksumFlag, + frameInfo.lz4FrameInfo.contentChecksumFlag); + if (totalBlocksSize) { + char bTypeBuffer[5]; + LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + bTypeBuffer, + frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-"); + if (frameInfo.lz4FrameInfo.contentSize) { + { double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100; + DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n", + totalBlocksSize + hSize, + frameInfo.lz4FrameInfo.contentSize, + ratio); + } + /* Now we've consumed frameInfo we can use it to store the total contentSize */ + frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize; + } + else { + DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-"); + cfinfo->allContentSize = 0; + } + result = LZ4IO_LZ4F_OK; + } + } + } + } + } + } + } + break; + case LEGACY_MAGICNUMBER: + frameInfo.frameType = legacyFrame; + if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput); + if (totalBlocksSize) { + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n", + cfinfo->frameCount + 1, + LZ4IO_frameTypeNames[frameInfo.frameType], + "-", "-", + totalBlocksSize + 4, + "-", "-"); + result = LZ4IO_LZ4F_OK; + } + } + break; + case LZ4IO_SKIPPABLE0: + frameInfo.frameType = skippableFrame; + if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0; + cfinfo->eqBlockTypes = 0; + cfinfo->allContentSize = 0; + { nbReadBytes = fread(buffer, 1, 4, finput); + if (nbReadBytes != 4) + EXM_THROW(42, "Stream error : skippable size unreadable"); + } + { unsigned const size = LZ4IO_readLE32(buffer); + int const errorNb = fseek_u32(finput, size, SEEK_CUR); + if (errorNb != 0) + EXM_THROW(43, "Stream error : cannot skip skippable area"); + DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n", + cfinfo->frameCount + 1, + "SkippableFrame", + "-", "-", size + 8, "-", "-"); + + result = LZ4IO_LZ4F_OK; + } + break; + default: + { long int const position = ftell(finput); /* only works for files < 2 GB */ + DISPLAYLEVEL(3, "Stream followed by undecodable data "); + if (position != -1L) + DISPLAYLEVEL(3, "at position %i ", (int)position); + DISPLAYLEVEL(3, "\n"); + } + break; + } + if (result != LZ4IO_LZ4F_OK) { + break; + } + cfinfo->frameSummary = frameInfo; + cfinfo->frameCount++; + } + fclose(finput); + return result; +} - DISPLAY("%5s %20s %20s %10s %7s %s\n", - "Block", "Compressed", "Uncompressed", "Ratio", "Check", "Filename"); - for (idx=0; idx<ifnIdx; idx++) { +int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx) +{ + int result = 0; + size_t idx = 0; + if (g_displayLevel < 3) { + DISPLAY("%10s %14s %5s %11s %13s %9s %s\n", + "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename"); + } + for (; idx < ifnIdx; idx++) { /* Get file info */ LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO; - LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]); - if (op_result != LZ4IO_LZ4F_OK) { - if (op_result == LZ4IO_not_a_file) { - DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]); - } else { + cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]); + if (!UTIL_isRegFile(inFileNames[idx])) { + DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]); + return 0; + } + DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx); + DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n", + "Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio") + { LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]); + if (op_result != LZ4IO_LZ4F_OK) { assert(op_result == LZ4IO_format_not_known); DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]); + return 0; } - result = 1; - continue; } - if (cfinfo.frameInfo.contentSize) { - char buffer[5]; - double const ratio = (double)cfinfo.fileSize / cfinfo.frameInfo.contentSize; - DISPLAY("%5s %20llu %20llu %8.4f %7s %s \n", - LZ4IO_blockTypeID(cfinfo.frameInfo.blockSizeID, cfinfo.frameInfo.blockMode, buffer), - cfinfo.fileSize, - cfinfo.frameInfo.contentSize, ratio, - cfinfo.frameInfo.contentChecksumFlag ? "XXH32" : "-", - cfinfo.fileName); - } else { - char buffer[5]; - DISPLAY("%5s %20llu %20s %10s %7s %s \n", - LZ4IO_blockTypeID(cfinfo.frameInfo.blockSizeID, cfinfo.frameInfo.blockMode, buffer), - cfinfo.fileSize, - "-", "-", - cfinfo.frameInfo.contentChecksumFlag ? "XXH32" : "-", - cfinfo.fileName); + DISPLAYLEVEL(3, "\n"); + if (g_displayLevel < 3) { + /* Display Summary */ + { char buffers[3][10]; + DISPLAY("%10llu %14s %5s %11s %13s ", + cfinfo.frameCount, + cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" , + cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID, + cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-", + LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]), + cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-"); + if (cfinfo.allContentSize) { + double const ratio = (double)cfinfo.fileSize / cfinfo.frameSummary.lz4FrameInfo.contentSize * 100; + DISPLAY("%9.2f%% %s \n", ratio, cfinfo.fileName); + } else { + DISPLAY("%9s %s\n", + "-", + cfinfo.fileName); + } + } } } + return result; } diff --git a/programs/lz4io.h b/programs/lz4io.h index 213dc95..b189e35 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -57,6 +57,8 @@ typedef struct LZ4IO_prefs_s LZ4IO_prefs_t; LZ4IO_prefs_t* LZ4IO_defaultPreferences(void); void LZ4IO_freePreferences(LZ4IO_prefs_t* const prefs); +/* Size in bytes of a legacy block header in little-endian format */ +#define LZIO_LEGACY_BLOCK_HEADER_SIZE 4 /* ************************************************** */ /* ****************** Functions ********************* */ diff --git a/tests/.gitignore b/tests/.gitignore index c4f9092..0d13df8 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -15,6 +15,7 @@ checkFrame # test artefacts tmp* versionsTest +lz4_all.c # local tests afl diff --git a/tests/Makefile b/tests/Makefile index 67514e4..422baba 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -114,7 +114,7 @@ clean: frametest$(EXT) frametest32$(EXT) \ fasttest$(EXT) roundTripTest$(EXT) \ datagen$(EXT) checkTag$(EXT) \ - frameTest$(EXT) + frameTest$(EXT) lz4_all.c @$(RM) -rf $(TESTDIR) @echo Cleaning completed @@ -122,10 +122,13 @@ clean: versionsTest: $(PYTHON) test-lz4-versions.py +.PHONY: listTest +listTest: lz4 + QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py + checkTag: checkTag.c $(LZ4DIR)/lz4.h $(CC) $(FLAGS) $< -o $@$(EXT) - #----------------------------------------------------------------------------- # validated only for Linux, OSX, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- @@ -150,19 +153,19 @@ list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs .PHONY: test -test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation +test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest .PHONY: test32 test32: CFLAGS+=-m32 test32: test -.PHONY: test-amalgamation -test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c - cat $(LZ4DIR)/lz4.c > lz4_all.c - cat $(LZ4DIR)/lz4hc.c >> lz4_all.c - cat $(LZ4DIR)/lz4frame.c >> lz4_all.c - $(CC) -I$(LZ4DIR) -c lz4_all.c - $(RM) lz4_all.c +test-amalgamation: lz4_all.o + +lz4_all.o: lz4_all.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@ + +lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c + cat $^ > $@ test-install: lz4 lib liblz4.pc lz4_root=.. ./test_install.sh @@ -373,6 +376,7 @@ test-lz4-testmode: lz4 datagen @echo "\n ---- non-existing source ----" ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist + ! $(LZ4) -t file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 @@ -444,7 +448,7 @@ test-fuzzer32: CFLAGS += -m32 test-fuzzer32: test-fuzzer test-frametest: frametest - ./frametest $(FUZZER_TIME) + ./frametest -v $(FUZZER_TIME) test-frametest32: CFLAGS += -m32 test-frametest32: test-frametest @@ -463,6 +467,8 @@ test-mem: lz4 datagen fuzzer frametest fullbench valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2 valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID) + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4 + valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4 $(RM) ftm* valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 diff --git a/tests/frametest.c b/tests/frametest.c index bf95beb..1b932e4 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -41,11 +41,12 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ #include <assert.h> -#include "lz4frame.h" /* include multiple times to test correctness/safety */ +#include "lz4frame.h" /* included multiple times to test correctness/safety */ #include "lz4frame.h" #define LZ4F_STATIC_LINKING_ONLY #include "lz4frame.h" #include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* LZ4_VERSION_STRING */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ @@ -150,8 +151,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou size_t const length = MIN(lengthRand, bufferSize - pos); size_t const end = pos + length; while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } + } } } @@ -541,7 +541,7 @@ int basicTests(U32 seed, double compressibility) cdict, NULL) ); DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", (unsigned)dictSize, (unsigned)cSizeWithDict); - if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */ + if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */ crcOrig = XXH64(CNBuffer, dictSize, 0); DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); @@ -703,8 +703,8 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { unsigned nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + size_t oSize = (size_t)(oend-op); + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -722,8 +722,8 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { unsigned const nbBits = FUZ_rand(&randState) % maxBits; size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; - size_t oSize = oend-op; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + size_t oSize = (size_t)(oend-op); + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -739,7 +739,7 @@ int basicTests(U32 seed, double compressibility) while (ip < iend) { size_t iSize = 10; size_t oSize = 10; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); op += oSize; ip += iSize; @@ -763,55 +763,173 @@ _output_error: } -static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) +typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e; + +static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario) { - size_t p=0; - const BYTE* b1=(const BYTE*)buff1; - const BYTE* b2=(const BYTE*)buff2; - DISPLAY("locateBuffDiff: looking for error position \n"); - if (nonContiguous) { - DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n", - nonContiguous, (unsigned)size); - return; + if (displayLevel >= 5) { + size_t p=0; + const BYTE* b1=(const BYTE*)buff1; + const BYTE* b2=(const BYTE*)buff2; + DISPLAY("locateBuffDiff: looking for error position \n"); + if (o_scenario != o_contiguous) { + DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n", + (int)o_scenario, (unsigned)size); + return; + } + while (p < size && b1[p]==b2[p]) p++; + if (p != size) { + DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]); + } } - while (p < size && b1[p]==b2[p]) p++; - if (p != size) { - DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]); +} + +# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); } +# undef CHECK +# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } } + + +size_t test_lz4f_decompression_wBuffers( + const void* cSrc, size_t cSize, + void* dst, size_t dstCapacity, o_scenario_e o_scenario, + const void* srcRef, size_t decompressedSize, + U64 crcOrig, + U32* const randState, + LZ4F_dctx* const dCtx, + U32 seed, U32 testNb) +{ + const BYTE* ip = (const BYTE*)cSrc; + const BYTE* const iend = ip + cSize; + + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstCapacity; + + unsigned const suggestedBits = FUZ_highbit((U32)cSize); + unsigned const maxBits = MAX(3, suggestedBits); + size_t totalOut = 0; + size_t moreToFlush = 0; + XXH64_state_t xxh64; + XXH64_reset(&xxh64, 1); + assert(ip < iend); + while (ip < iend) { + unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1; + unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1; + size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1; + size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip)); + size_t iSize = iSizeMax; + size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2; + size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op)); + size_t oSize = oSizeMax; + BYTE const mark = (BYTE)(FUZ_rand(randState) & 255); + LZ4F_decompressOptions_t dOptions; + memset(&dOptions, 0, sizeof(dOptions)); + dOptions.stableDst = FUZ_rand(randState) & 1; + if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */ + if (op + oSizeMax < oend) op[oSizeMax] = mark; + + DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize); + + /* read data from byte-exact buffer to catch out-of-bound reads */ + { void* const iBuffer = malloc(iSizeMax); + assert(iBuffer != NULL); + memcpy(iBuffer, ip, iSizeMax); + moreToFlush = LZ4F_decompress(dCtx, op, &oSize, iBuffer, &iSize, &dOptions); + free(iBuffer); + } + DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize); + + if (op + oSizeMax < oend) { + CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : " + "Decompression overwrites beyond assigned dst size", + op[oSizeMax], mark); + } + if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) + locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); + if (LZ4F_isError(moreToFlush)) return moreToFlush; + + XXH64_update(&xxh64, op, oSize); + totalOut += oSize; + op += oSize; + ip += iSize; + if (o_scenario == o_noncontiguous) { + if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */ + op++; /* create a gap between consecutive output */ + } + if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */ + if ( (op == oend) /* no more room for output; can happen with bogus input */ + && (iSize == 0)) /* no input consumed */ + break; } + if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed; + if (totalOut) { /* otherwise, it's a skippable frame */ + U64 const crcDecoded = XXH64_digest(&xxh64); + if (crcDecoded != crcOrig) { + locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); + return LZ4F_ERROR_contentChecksum_invalid; + } } + return 0; +} + + +size_t test_lz4f_decompression(const void* cSrc, size_t cSize, + const void* srcRef, size_t decompressedSize, + U64 crcOrig, + U32* const randState, + LZ4F_dctx* const dCtx, + U32 seed, U32 testNb) +{ + o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ + /* tighten dst buffer conditions */ + size_t const dstCapacity = (o_scenario == o_noncontiguous) ? + (decompressedSize * 2) + 128 : + decompressedSize; + size_t result; + void* const dstBuffer = malloc(dstCapacity); + assert(dstBuffer != NULL); + + result = test_lz4f_decompression_wBuffers(cSrc, cSize, + dstBuffer, dstCapacity, o_scenario, + srcRef, decompressedSize, + crcOrig, + randState, + dCtx, + seed, testNb); + + free(dstBuffer); + return result; } int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) { - int testResult = 0; unsigned testNb = 0; - size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - void* srcBuffer = NULL; - size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL) + 4 MB; /* needs some margin */ + size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ + void* CNBuffer = NULL; + size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */ void* compressedBuffer = NULL; void* decodedBuffer = NULL; U32 coreRand = seed; LZ4F_decompressionContext_t dCtx = NULL; + LZ4F_decompressionContext_t dCtxNoise = NULL; LZ4F_compressionContext_t cCtx = NULL; clock_t const startClock = clock(); clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; -# undef CHECK -# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } -# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } } /* Create buffers */ { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } + { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION); + CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } - srcBuffer = malloc(srcDataLength); - CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); + CNBuffer = malloc(CNBufferLength); + CHECK(CNBuffer==NULL, "CNBuffer Allocation failed"); compressedBuffer = malloc(compressedBufferSize); CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ + decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); - FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand); /* jump to requested testNb */ for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ @@ -819,10 +937,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi /* main fuzzer test loop */ for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { U32 randState = coreRand ^ prime1; - unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1; + unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1; size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; - size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize); - const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId; + size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize); + const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId; unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; U64 const crcOrig = XXH64(srcStart, srcSize, 1); LZ4F_preferences_t prefs; @@ -848,9 +966,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); FUZ_writeLE32(op+4, (U32)srcSize); cSize = srcSize+8; + } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); + } else { /* multi-segments compression */ const BYTE* ip = srcStart; const BYTE* const iend = srcStart + srcSize; @@ -913,58 +1033,53 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); } + /* multi-segments decompression */ - { const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = ip + cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = op + srcDataLength; - unsigned const suggestedBits = FUZ_highbit((U32)cSize); - unsigned const maxBits = MAX(3, suggestedBits); - unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ - size_t totalOut = 0; - size_t decSize = 0; - XXH64_state_t xxh64; - XXH64_reset(&xxh64, 1); - assert(ip < iend); - while (ip < iend) { - unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; - unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1; - size_t iSize = MIN(iSizeMax, (size_t)(iend-ip)); - size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2; - size_t oSize = MIN(oSizeMax, (size_t)(oend-op)); - LZ4F_decompressOptions_t dOptions; - memset(&dOptions, 0, sizeof(dOptions)); - dOptions.stableDst = FUZ_rand(&randState) & 1; - if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */ - decSize = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (LZ4F_getErrorCode(decSize) == LZ4F_ERROR_contentChecksum_invalid) - locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LZ4F_isError(decSize), "Decompression failed (error %i:%s)", - (int)decSize, LZ4F_getErrorName(decSize)); - XXH64_update(&xxh64, op, (U32)oSize); - totalOut += oSize; - op += oSize; - ip += iSize; - op += nonContiguousDst; - if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ - } - CHECK(decSize != 0, "Frame decompression failed (error %i)", (int)decSize); - if (totalOut) { /* otherwise, it's a skippable frame */ - U64 const crcDecoded = XXH64_digest(&xxh64); - if (crcDecoded != crcOrig) { - locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst); - EXIT_MSG("Decompression corruption"); - } } + DISPLAYLEVEL(6, "normal decompression \n"); + { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb); + CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)", + (int)result, LZ4F_getErrorName(result)); } - } + +#if 1 + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit((U32)cSize); + size_t pos = 0; + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&randState) % maxNbBits; + size_t const mask = (1<<nbBits) - 1; + size_t const skipLength = FUZ_rand(&randState) & mask; + pos += skipLength; + } + if (pos >= cSize) break; + /* add noise */ + { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<<nbBits) - 1; + size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1; + size_t const noiseLength = MIN(rNoiseLength, cSize-pos); + size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength); + memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } + + /* test decompression on noisy src */ + DISPLAYLEVEL(6, "noisy decompression \n"); + test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb); + /* note : we don't analyze result here : it probably failed, which is expected. + * We just check for potential out-of-bound reads and writes. */ + LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */ +#endif + +} /* for ( ; (testNb < nbTests) ; ) */ DISPLAYLEVEL(2, "\rAll tests completed \n"); -_end: LZ4F_freeDecompressionContext(dCtx); + LZ4F_freeDecompressionContext(dCtxNoise); LZ4F_freeCompressionContext(cCtx); - free(srcBuffer); + free(CNBuffer); free(compressedBuffer); free(decodedBuffer); @@ -972,11 +1087,7 @@ _end: DISPLAY("press enter to finish \n"); (void)getchar(); } - return testResult; - -_output_error: - testResult = 1; - goto _end; + return 0; } @@ -1002,8 +1113,8 @@ int main(int argc, const char** argv) U32 seed=0; int seedset=0; int argNb; - int nbTests = nbTestsDefault; - int testNb = 0; + unsigned nbTests = nbTestsDefault; + unsigned testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int result=0; U32 duration=0; @@ -1048,7 +1159,7 @@ int main(int argc, const char** argv) nbTests=0; duration=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; - nbTests += *argument - '0'; + nbTests += (unsigned)(*argument - '0'); argument++; } break; @@ -1071,7 +1182,7 @@ int main(int argc, const char** argv) case '6': case '7': case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; + case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue; } break; } @@ -1083,7 +1194,7 @@ int main(int argc, const char** argv) seedset=1; while ((*argument>='0') && (*argument<='9')) { seed *= 10; - seed += *argument - '0'; + seed += (U32)(*argument - '0'); argument++; } break; @@ -1092,7 +1203,7 @@ int main(int argc, const char** argv) testNb=0; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; - testNb += *argument - '0'; + testNb += (unsigned)(*argument - '0'); argument++; } break; @@ -1126,7 +1237,7 @@ int main(int argc, const char** argv) DISPLAY("Seed = %u\n", seed); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); - if (nbTests<=0) nbTests=1; + nbTests += (nbTests==0); /* avoid zero */ if (testNb==0) result = basicTests(seed, ((double)proba) / 100); if (result) return 1; diff --git a/tests/fullbench.c b/tests/fullbench.c index d2af662..7d74d3f 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -343,11 +343,44 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS assert(inSize >= 0); assert(outSize >= 0); result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); - if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); } - if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } + if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); } + if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); } return (int)dstSize; } +static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize) +{ + size_t totalInSize = (size_t)srcSize; + size_t maxOutSize = (size_t)dstSize; + + size_t inPos = 0; + size_t inSize = 0; + size_t outPos = 0; + size_t outRemaining = maxOutSize - outPos; + + for (;;) { + size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL); + assert(!LZ4F_isError(sizeHint)); + + inPos += inSize; + inSize = sizeHint; + + outPos += outRemaining; + outRemaining = maxOutSize - outPos; + + if (!sizeHint) break; + } + + /* frame completed */ + if (inPos != totalInSize) { + DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n", + (unsigned)inPos, (unsigned)totalInSize); + exit(10); + } + return (int)outPos; + +} + #define NB_COMPRESSION_ALGORITHMS 100 #define NB_DECOMPRESSION_ALGORITHMS 100 @@ -446,7 +479,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) for (i=0; i<nbChunks; i++) { chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + assert(g_chunkSize > 0); + if (remaining > (size_t)g_chunkSize) { + chunkP[i].origSize = g_chunkSize; + remaining -= (size_t)g_chunkSize; + } else { + chunkP[i].origSize = (int)remaining; + remaining = 0; + } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } @@ -501,9 +541,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) if (initFunction!=NULL) initFunction(); for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); - if (chunkP[chunkNb].compressedSize==0) - DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); - } + if (chunkP[chunkNb].compressedSize==0) { + DISPLAY("ERROR ! %s() = 0 !! \n", compressorName); + exit(1); + } } nb_loops++; } clockTime = BMK_GetClockSpan(clockTime); @@ -511,7 +552,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) nb_loops += !nb_loops; /* avoid division by zero */ averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC; if (averageTime < bestTime) bestTime = averageTime; - cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; + cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += (size_t)chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); } @@ -531,23 +572,30 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { - chunkP[i].id = i; + chunkP[i].id = (U32)i; chunkP[i].origBuffer = in; in += g_chunkSize; - if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + if ((int)remaining > g_chunkSize) { + chunkP[i].origSize = g_chunkSize; + remaining -= (size_t)g_chunkSize; + } else { + chunkP[i].origSize = (int)remaining; + remaining = 0; + } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } } for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize); - if (chunkP[chunkNb].compressedSize==0) - DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); - } + if (chunkP[chunkNb].compressedSize==0) { + DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"); + exit(1); + } } /* Decompression Algorithms */ for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) { - const char* dName; - int (*decompressionFunction)(const char*, char*, int, int); + const char* dName = NULL; + int (*decompressionFunction)(const char*, char*, int, int) = NULL; double bestTime = 100000000.; int checkResult = 1; @@ -565,17 +613,15 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) #ifndef LZ4_DLL_IMPORT case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; #endif - case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; - { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); - if (LZ4F_isError(errorCode)) { - DISPLAY("Error while preparing compressed frame\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 1; - } + case 10: + case 11: + if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */ + if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */ + /* prepare compressed data using frame format */ + { size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL); + assert(!LZ4F_isError(fcsize)); chunkP[0].origSize = (int)benchedSize; - chunkP[0].compressedSize = (int)errorCode; + chunkP[0].compressedSize = (int)fcsize; nbChunks = 1; break; } @@ -583,6 +629,9 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles) continue; /* skip if unknown ID */ } + assert(decompressionFunction != NULL); + assert(dName != NULL); + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */ for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a5b5c93..8a095c4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -122,6 +122,14 @@ static U32 FUZ_rotl32(U32 u32, U32 nbBits) return ((u32 << nbBits) | (u32 >> (32 - nbBits))); } +static U32 FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + while (v32) { v32 >>= 1; nbBits++; } + return nbBits; +} + static U32 FUZ_rand(U32* src) { U32 rand32 = *src; @@ -207,7 +215,7 @@ static int FUZ_AddressOverflow(void) } { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512); - unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1); + int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1); char* const input = buffers[nbBuff-1]; char* output = buffers[nbBuff]; int r; @@ -215,7 +223,7 @@ static int FUZ_AddressOverflow(void) input[1] = (char)0xFF; input[2] = (char)0xFF; input[3] = (char)0xFF; - { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } + { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; } r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134); if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; } input[0] = (char)0x1F; /* Match length overflow */ @@ -366,7 +374,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c U32 testNb = 0; U32 randState = FUZ_rand(&coreRandState) ^ PRIME3; int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; + int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1; int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; int const dictSize = MIN(dictSizeRand, blockStart - 1); int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1); @@ -388,69 +396,65 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression destSize */ FUZ_DISPLAYTEST("test LZ4_compress_destSize()"); - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char endCheck = FUZ_rand(&randState) & 255; + { int cSize, srcSize = blockSize; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); + cSize = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); + FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); - DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() read more than src buffer !"); + DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", cSize, targetSize, srcSize, blockSize); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); + char const canary = (char)(FUZ_rand(&randState) & 255); + FUZ_CHECKTEST((cSize==0), "LZ4_compress_destSize() compression failed"); FUZ_DISPLAYTEST(); - compressedSize = ret; decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize); + FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); + FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + } FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); - FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); } - - DISPLAYLEVEL(5, " OK \n"); - } else { - DISPLAYLEVEL(5, " \n"); - } } + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); + FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); + } } + DISPLAYLEVEL(5, " OK \n"); + } /* Test compression HC destSize */ FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()"); - { int srcSize = blockSize; - int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; - char const endCheck = FUZ_rand(&randState) & 255; - void* ctx = LZ4_createHC(block); + { int cSize, srcSize = blockSize; + int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7; + char const endCheck = (char)(FUZ_rand(&randState) & 255); + void* const ctx = LZ4_createHC(block); FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); compressedBuffer[targetSize] = endCheck; - ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); + cSize = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel); DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ", - compressionLevel, ret, targetSize, srcSize, blockSize); + compressionLevel, cSize, targetSize, srcSize, blockSize); LZ4_freeHC(ctx); - FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !"); FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !"); FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !"); if (targetSize>0) { /* check correctness */ U32 const crcBase = XXH32(block, (size_t)srcSize, 0); - char const canary = FUZ_rand(&randState) & 255; - FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed"); + char const canary = (char)(FUZ_rand(&randState) & 255); + FUZ_CHECKTEST((cSize==0), "LZ4_compress_HC_destSize() compression failed"); FUZ_DISPLAYTEST(); - compressedSize = ret; decodedBuffer[srcSize] = canary; - ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); - FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize); + FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize"); + FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + } FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !"); - { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0); + { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0); FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); - } - DISPLAYLEVEL(5, " OK \n"); - } else { - DISPLAYLEVEL(5, " \n"); - } } + } } + DISPLAYLEVEL(5, " OK \n"); + } /* Test compression HC */ FUZ_DISPLAYTEST("test LZ4_compress_HC()"); @@ -524,7 +528,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data"); } FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data"); } @@ -559,13 +563,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size"); } + /* noisy src decompression test */ + + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit32((U32)compressedSize); + size_t pos = 0; + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&randState) % maxNbBits; + size_t const mask = (1<<nbBits) - 1; + size_t const skipLength = FUZ_rand(&randState) & mask; + pos += skipLength; + } + if (pos >= (size_t)compressedSize) break; + /* add noise */ + { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<<nbBits) - 1; + size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1; + size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos); + size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength); + memcpy(cBuffer_exact + pos, (const char*)CNBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } + + /* decompress noisy source */ + FUZ_DISPLAYTEST("decompress noisy source "); + { U32 const endMark = 0xA9B1C3D6; + memcpy(decodedBuffer+blockSize, &endMark, sizeof(endMark)); + { int const decompressResult = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize); + /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ + FUZ_CHECKTEST(decompressResult > blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize); + } + { U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck)); + FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow"); + } } /* noisy src decompression test */ + free(cBuffer_exact); } /* Test decoding with input size being one byte too short => must fail */ FUZ_DISPLAYTEST(); { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); + FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize); } /* Test decoding with input size being one byte too large => must fail */ @@ -578,7 +618,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test partial decoding => must work */ FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial"); - { size_t const missingBytes = FUZ_rand(&randState) % blockSize; + { size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize; int const targetSize = (int)((size_t)blockSize - missingBytes); char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A; int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize); @@ -615,8 +655,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= compressedSize) missingBytes = compressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[compressedSize-missingBytes] = 0; - ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); + { int const cSize = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes); + FUZ_CHECKTEST(cSize, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes); + } FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes) } @@ -626,8 +667,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); - FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); + { int const hcSize = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel); + FUZ_CHECKTEST(hcSize, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes); + } FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes) } @@ -654,8 +696,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()"); ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); @@ -672,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed"); - FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte"); + FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte"); LZ4_loadDict(&LZ4dictBody, dict, dictSize); ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1); FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); @@ -697,15 +738,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } @@ -722,17 +762,16 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST(); - { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) { + { int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; + if (blockSize > missingBytes) { decodedBuffer[blockSize-missingBytes] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes); + FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize); } } /* Compress using external dictionary stream */ - { - LZ4_stream_t LZ4_stream; + { LZ4_stream_t LZ4_stream; int expectedSize; U32 expectedCrc; @@ -740,7 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_loadDict(&LZ4dictBody, dict, dictSize); expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1); FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded"); - expectedCrc = XXH32(compressedBuffer, expectedSize, 0); + expectedCrc = XXH32(compressedBuffer, (size_t)expectedSize, 0); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()"); LZ4_loadDict(&LZ4dictBody, dict, dictSize); @@ -756,7 +795,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c * test. */ FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize); - FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short"); LZ4_resetStream_fast(&LZ4_stream); @@ -772,7 +811,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); - FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); FUZ_DISPLAYTEST(); @@ -782,7 +821,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer"); FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output"); - FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); + FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output"); FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good"); } @@ -792,12 +831,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input"); FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); - { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0); if (crcCheck!=crcOrig) { FUZ_findDiff(block, decodedBuffer); EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); - } - } + } } FUZ_DISPLAYTEST(); decodedBuffer[blockSize] = 0; @@ -823,7 +861,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer"); { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; if ((U32)blockSize > missingBytes) { - decodedBuffer[blockSize-missingBytes] = 0; + decodedBuffer[(U32)blockSize-missingBytes] = 0; ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); @@ -965,8 +1003,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c } -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) +#define testInputSize (196 KB) +#define testCompressedSize (130 KB) #define ringBufferSize (8 KB) static void FUZ_unitTests(int compressionLevel) @@ -990,17 +1028,17 @@ static void FUZ_unitTests(int compressionLevel) FUZ_AddressOverflow(); /* Test decoding with empty input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n"); LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize); /* Test decoding with a one byte input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n"); { char const tmp = (char)0xFF; LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize); } /* Test decoding shortcut edge case */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case"); + DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n"); { char tmp[17]; /* 14 bytes of literals, followed by a 14 byte match. * Should not read beyond the end of the buffer. @@ -1013,6 +1051,54 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail"); } } + /* in-place compression test */ + DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :"); + { int const sampleSize = 65 KB; + int const maxCSize = LZ4_COMPRESSBOUND(sampleSize); + int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize); + int const startInputIndex = outSize - sampleSize; + char* const startInput = testCompressed + startInputIndex; + XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0); + int cSize; + assert(outSize < (int)testCompressedSize); + memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */ + /* compress in-place */ + cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize); + assert(cSize != 0); /* ensure compression is successful */ + assert(maxCSize < INT_MAX); + assert(cSize <= maxCSize); + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); + assert(dSize == sampleSize); /* correct size */ + { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } + DISPLAYLEVEL(3, " OK \n"); + + /* in-place decompression test */ + DISPLAYLEVEL(3, "in-place decompression, limit case:"); + { int const sampleSize = 65 KB; + + FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState); + memset(testInput, 0, 267); /* calculated exactly so that compressedSize == originalSize-1 */ + + { XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0); + int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize); + assert(cSize == sampleSize - 1); /* worst case for in-place decompression */ + + { int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize); + int const startInputIndex = bufferSize - cSize; + char* const startInput = testVerify + startInputIndex; + memcpy(startInput, testCompressed, cSize); + + /* decompress and verify */ + { int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize); + assert(dSize == sampleSize); /* correct size */ + { XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0); + assert(crcCheck == crcOrig); + } } } } } + DISPLAYLEVEL(3, " OK \n"); + /* LZ4 streaming tests */ { LZ4_stream_t streamingState; U64 crcOrig; @@ -1061,17 +1147,17 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1); + compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1151,12 +1237,12 @@ static void FUZ_unitTests(int compressionLevel) { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + } } { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } } @@ -1165,7 +1251,8 @@ static void FUZ_unitTests(int compressionLevel) /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); + XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) , + XXH64(testInput + segSize, testCompressedSize, 0) ); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -1179,7 +1266,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } } @@ -1201,15 +1288,15 @@ static void FUZ_unitTests(int compressionLevel) { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); + size_t dictSize = (FUZ_rand(&randState) & 8191); char* dst = testVerify; - size_t segStart = (size_t)dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); + size_t segStart = dictSize + 7; + size_t segSize = (FUZ_rand(&randState) & 8191); int segNb = 1; LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, dict, dictSize); + LZ4_loadDictHC(&sHC, dict, (int)dictSize); XXH64_reset(&crcOrigState, 0); XXH64_reset(&crcNewState, 0); @@ -1217,29 +1304,28 @@ static void FUZ_unitTests(int compressionLevel) while (segStart + segSize < testInputSize) { XXH64_update(&crcOrigState, testInput + segStart, segSize); crcOrig = XXH64_digest(&crcOrigState); - result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize)); + assert(segSize <= INT_MAX); + result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize)); FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize); + FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb); XXH64_update(&crcNewState, dst, segSize); { U64 const crcNew = XXH64_digest(&crcNewState); if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb); } - assert(segSize >= 0); dict = dst; dictSize = segSize; - dst += (size_t)segSize + 1; + dst += segSize + 1; segNb ++; - segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1; + segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; segSize = (FUZ_rand(&randState) & 8191); - } - } + } } /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -1266,18 +1352,21 @@ static void FUZ_unitTests(int compressionLevel) crcOrig = XXH64_digest(&xxhOrig); memcpy (ringBuffer + rNext, testInput + iNext, messageSize); - compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + assert(messageSize < INT_MAX); + compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed"); FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize); + assert(messageSize < INT_MAX); + result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); @@ -1337,14 +1426,14 @@ static void FUZ_unitTests(int compressionLevel) result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize); FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); } @@ -1360,7 +1449,7 @@ static void FUZ_unitTests(int compressionLevel) dNext = 0; while (totalMessageSize < 9 MB) { - XXH64_update(&xxhOrig, testInput + iNext, messageSize); + XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize); crcOrig = XXH64_digest(&xxhOrig); compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); @@ -1375,7 +1464,7 @@ static void FUZ_unitTests(int compressionLevel) testCompressed, ringBufferSafe + dNext, compressedSize, dBufferSize - dNext); /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */ FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed"); - XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize); + XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test"); @@ -1384,7 +1473,7 @@ static void FUZ_unitTests(int compressionLevel) /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */ result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize); FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed"); - XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize); + XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test"); @@ -1392,7 +1481,8 @@ static void FUZ_unitTests(int compressionLevel) /* prepare next message */ dNext += messageSize; - totalMessageSize += messageSize; + assert(messageSize >= 0); + totalMessageSize += (unsigned)messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; iNext = (FUZ_rand(&randState) & 65535); if (dNext + maxMessageSize > dBufferSize) dNext = 0; @@ -1410,6 +1500,11 @@ static void FUZ_unitTests(int compressionLevel) } + +/* ======================================= + * CLI + * ======================================= */ + static int FUZ_usage(const char* programName) { DISPLAY( "Usage :\n"); @@ -1433,8 +1528,8 @@ int main(int argc, const char** argv) U32 seed = 0; int seedset = 0; int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; + unsigned nbTests = NB_ATTEMPTS; + unsigned testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int use_pause = 0; const char* programName = argv[0]; @@ -1472,7 +1567,7 @@ int main(int argc, const char** argv) nbTests = 0; duration = 0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; - nbTests += *argument - '0'; + nbTests += (unsigned)(*argument - '0'); argument++; } break; @@ -1495,7 +1590,7 @@ int main(int argc, const char** argv) case '6': case '7': case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; + case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue; } break; } @@ -1506,7 +1601,7 @@ int main(int argc, const char** argv) seed=0; seedset=1; while ((*argument>='0') && (*argument<='9')) { seed *= 10; - seed += *argument - '0'; + seed += (U32)(*argument - '0'); argument++; } break; @@ -1516,7 +1611,7 @@ int main(int argc, const char** argv) testNb=0; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; - testNb += *argument - '0'; + testNb += (unsigned)(*argument - '0'); argument++; } break; @@ -1551,7 +1646,7 @@ int main(int argc, const char** argv) if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); } - if (nbTests<=0) nbTests=1; + nbTests += (nbTests==0); /* avoid zero */ { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); if (use_pause) { diff --git a/tests/test-lz4-list.py b/tests/test-lz4-list.py new file mode 100644 index 0000000..ce89757 --- /dev/null +++ b/tests/test-lz4-list.py @@ -0,0 +1,282 @@ +#! /usr/bin/env python3 +import subprocess +import time +import glob +import os +import tempfile +import unittest + +SIZES = [3, 11] # Always 2 sizes +MIB = 1048576 +LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4" +if not os.path.exists(LZ4): + LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4" +TEMP = tempfile.gettempdir() + + +class NVerboseFileInfo(object): + def __init__(self, line_in): + self.line = line_in + splitlines = line_in.split() + if len(splitlines) != 7: + errout("Unexpected line: {}".format(line_in)) + self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines + self.exp_unc_size = 0 + # Get real file sizes + if "concat-all" in self.filename or "2f--content-size" in self.filename: + for i in SIZES: + self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i)) + else: + uncompressed_filename = self.filename.split("-")[0] + self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename)) + self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename)) + + +class TestNonVerbose(unittest.TestCase): + @classmethod + def setUpClass(self): + self.nvinfo_list = [] + for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)): + if i > 0: + self.nvinfo_list.append(NVerboseFileInfo(line)) + + def test_frames(self): + all_concat_frames = 0 + all_concat_index = None + for i, nvinfo in enumerate(self.nvinfo_list): + if "concat-all" in nvinfo.filename: + all_concat_index = i + elif "2f--content-size" in nvinfo.filename: + self.assertEqual("2", nvinfo.frames, nvinfo.line) + all_concat_frames += 2 + else: + self.assertEqual("1", nvinfo.frames, nvinfo.line) + all_concat_frames += 1 + self.assertNotEqual(None, all_concat_index, "Couldn't find concat-all file index.") + self.assertEqual(self.nvinfo_list[all_concat_index].frames, str(all_concat_frames), self.nvinfo_list[all_concat_index].line) + + def test_frame_types(self): + for nvinfo in self.nvinfo_list: + if "-lz4f-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LZ4Frame", nvinfo.line) + elif "-legc-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "LegacyFrame", nvinfo.line) + elif "-skip-" in nvinfo.filename: + self.assertEqual(nvinfo.type, "SkippableFrame", nvinfo.line) + + def test_block(self): + for nvinfo in self.nvinfo_list: + # if "-leg" in nvinfo.filename or "-skip" in nvinfo.filename: + # self.assertEqual(nvinfo.block, "-", nvinfo.line) + if "--BD" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+D$", nvinfo.line) + elif "--BI" in nvinfo.filename: + self.assertRegex(nvinfo.block, "^B[0-9]+I$", nvinfo.line) + + def test_compressed_size(self): + for nvinfo in self.nvinfo_list: + self.assertEqual(nvinfo.compressed, to_human(nvinfo.exp_comp_size), nvinfo.line) + + def test_ratio(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line) + + def test_uncompressed_size(self): + for nvinfo in self.nvinfo_list: + if "--content-size" in nvinfo.filename: + self.assertEqual(nvinfo.uncompressed, to_human(nvinfo.exp_unc_size), nvinfo.line) + + +class VerboseFileInfo(object): + def __init__(self, lines): + # Parse lines + self.frame_list = [] + self.file_frame_map = [] + for i, line in enumerate(lines): + if i == 0: + self.filename = line + continue + elif i == 1: + # Skip header + continue + frame_info = dict(zip(["frame", "type", "block", "checksum", "compressed", "uncompressed", "ratio"], line.split())) + frame_info["line"] = line + self.frame_list.append(frame_info) + + +class TestVerbose(unittest.TestCase): + @classmethod + def setUpClass(self): + # Even do we're listing 2 files to test multiline working as expected. + # we're only really interested in testing the output of the concat-all file. + self.vinfo_list = [] + start = end = 0 + output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True) + for i, line in enumerate(output): + if line.startswith("test_list"): + if start != 0 and end != 0: + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + start = i + if not line: + end = i + self.vinfo_list.append(VerboseFileInfo(output[start:end])) + # Populate file_frame_map as a reference of the expected info + concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4") + # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file + for i, filename in enumerate(concat_file_list): + if "2f--content-size" in filename: + concat_file_list.insert(i, filename) + break + self.cvinfo = self.vinfo_list[0] + self.cvinfo.file_frame_map = concat_file_list + self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP)) + + def test_filename(self): + for i, vinfo in enumerate(self.vinfo_list): + self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list))) + + def test_frame_number(self): + for vinfo in self.vinfo_list: + for i, frame_info in enumerate(vinfo.frame_list): + self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"]) + + def test_frame_type(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"]) + elif "-legc-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"]) + elif "-skip-" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"]) + + def test_block(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--BD" in self.cvinfo.file_frame_map[i]: + self.assertRegex(self.cvinfo.frame_list[i]["block"], "^B[0-9]+D$", self.cvinfo.frame_list[i]["line"]) + elif "--BI" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["block"], "^B[0-9]+I$", self.cvinfo.frame_list[i]["line"]) + + def test_checksum(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"]) + + def test_compressed(self): + total = 0 + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "-2f-" not in self.cvinfo.file_frame_map[i]: + expected_size = os.path.getsize(self.cvinfo.file_frame_map[i]) + self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"]) + total += int(self.cvinfo.frame_list[i]["compressed"]) + self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename)) + + def test_uncompressed(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + ffm = self.cvinfo.file_frame_map[i] + if "-2f-" not in ffm and "--content-size" in ffm: + expected_size_unc = int(ffm[ffm.rindex("_") + 1:ffm.index("M")]) * 1048576 + self.assertEqual(self.cvinfo.frame_list[i]["uncompressed"], str(expected_size_unc), self.cvinfo.frame_list[i]["line"]) + + def test_ratio(self): + for i, frame_info in enumerate(self.cvinfo.frame_list): + if "--content-size" in self.cvinfo.file_frame_map[i]: + self.assertEqual(self.cvinfo.frame_list[i]['ratio'], + "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100), + self.cvinfo.frame_list[i]["line"]) + + +def to_human(size): + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']: + if size < 1024.0: + break + size /= 1024.0 + return "{:.2f}{}".format(size, unit) + + +def log(text): + print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) + + +def errout(text, err=1): + log(text) + exit(err) + + +def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): + if os.environ.get('QEMU_SYS'): + command = "{} {}".format(os.environ['QEMU_SYS'], command) + if print_command: + log("> " + command) + popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell) + stdout_lines, stderr_lines = popen.communicate() + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") + if print_output: + if stdout_lines: + print(stdout_lines) + if stderr_lines: + print(stderr_lines) + if popen.returncode is not None and popen.returncode != 0: + if stderr_lines and not print_output and print_error: + print(stderr_lines) + errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines)) + return (stdout_lines + stderr_lines).splitlines() + + +def cleanup(silent=False): + for f in glob.glob("{}/test_list*".format(TEMP)): + if not silent: + log("Deleting {}".format(f)) + os.unlink(f) + + +def datagen(file_name, size): + non_sparse_size = size // 2 + sparse_size = size - non_sparse_size + with open(file_name, "wb") as f: + f.seek(sparse_size) + f.write(os.urandom(non_sparse_size)) + + +def generate_files(): + # file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~ + # Generate LZ4Frames + for i in SIZES: + filename = "{}/test_list_{}M".format(TEMP, i) + log("Generating {}".format(filename)) + datagen(filename, i * MIB) + for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]: + lz4file = "{}-lz4f-1f{}.lz4".format(filename, j) + execute("{} {} {} {}".format(LZ4, j, filename, lz4file)) + # Generate skippable frames + lz4file = "{}-skip-1f.lz4".format(filename) + skipsize = i * 1024 + skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False) + with open(lz4file, 'wb') as f: + f.write(skipbytes) + f.write(os.urandom(skipsize)) + # Generate legacy frames + lz4file = "{}-legc-1f.lz4".format(filename) + execute("{} -l {} {}".format(LZ4, filename, lz4file)) + + # Concatenate --content-size files + file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP)) + with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + # Concatenate all files + file_list = glob.glob("{}/test_list_*.lz4".format(TEMP)) + with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile: + for fname in file_list: + with open(fname, 'rb') as infile: + outfile.write(infile.read()) + + +if __name__ == '__main__': + cleanup() + generate_files() + unittest.main(verbosity=2, exit=False) + cleanup(silent=True) |