diff options
-rw-r--r-- | .github/workflows/README.md | 73 | ||||
-rw-r--r-- | .github/workflows/ci.yml | 690 | ||||
-rw-r--r-- | .travis.yml | 398 | ||||
-rw-r--r-- | Makefile | 13 | ||||
-rw-r--r-- | examples/blockStreaming_lineByLine.c | 2 | ||||
-rw-r--r-- | examples/dictionaryRandomAccess.c | 2 | ||||
-rw-r--r-- | lib/lz4.c | 86 | ||||
-rw-r--r-- | lib/lz4frame.c | 121 | ||||
-rw-r--r-- | lib/lz4frame.h | 6 | ||||
-rw-r--r-- | tests/checkFrame.c | 2 | ||||
-rw-r--r-- | tests/frametest.c | 5 |
11 files changed, 1100 insertions, 298 deletions
diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..eddfd3f --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,73 @@ +This directory contains [GitHub Actions](https://github.com/features/actions) workflow files. + +# Maintenance Schedule + +[`ubuntu-16.04` environment will be removed at September, 2021.]((https://github.blog/changelog/2021-04-29-github-actions-ubuntu-16-04-lts-virtual-environment-will-be-removed-on-september-20-2021/). +---------------------------------------------------------------- + +We also will remove test for the following compilers. + +- gcc: 4.4, 4.6, 4.7 +- clang: 3.5, 3.6, 3.7, 3.8 + + + +# Known issues + +## USAN, ASAN (`lz4-ubsan-x64`, `lz4-ubsan-x86`, `lz4-asan-x64`) + +For now, `lz4-ubsan-*` ignores the exit code of `make usan` and `make usan32`. +Because there're several issues which may take relatively long time to resolve. + +We'll fully enable it when we ensure `make usan` is ready for all commits and PRs. + +See [#983](https://github.com/lz4/lz4/pull/983) for details. + + +## C Compilers (`lz4-c-compilers`) + +- Our test doesn't use `gcc-4.5` due to installation issue of its package. (`apt-get install gcc-4.5` fails on GH-Actions VM) + +- Currently, the following 32bit executable tests fail with all versions of `clang`. + - `CC=clang-X CFLAGS='-O3' make V=1 -C tests clean test-lz4c32` + - `CC=clang-X CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32` + - See [#991](https://github.com/lz4/lz4/issues/991) for details. + +- Currently, the following 32bit executable tests fail with `gcc-11` + - `CC=gcc-11 CFLAGS='-O3' make V=1 -C tests clean test-lz4c32` + - `CC=gcc-11 CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32` + - See [#991](https://github.com/lz4/lz4/issues/991) for details. + + +## cppcheck (`lz4-cppcheck`) + +This test script ignores the exit code of `make cppcheck`. +Because this project doesn't 100% follow their recommendation. +Also sometimes it reports false positives. + + + +# Notes + +- You can investigate various information at the right pane of GitHub + Actions report page. + +| Item | Section in the right pane | +| ------------------------- | ------------------------------------- | +| OS, VM | Set up job | +| git repo, commit hash | Run actions/checkout@v2 | +| Version of tools | Environment info | + + + +# Difference with `.travis.yml` + +The following tests are not included yet. + +- name: Compile OSS-Fuzz targets + +The following tests will not be included due to limitation of GH-Actions. + +- name: aarch64 real-hw tests +- name: PPC64LE real-hw tests +- name: IBM s390x real-hw tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b8e3bb5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,690 @@ +# For details, see README.md in this directory. + +############################################################### +# C compilers +# +# - gcc +# - clang +# +# Known Issue +# - All test cases which described as 'fail' must be fixed and replaced with 'true'. +# - gcc-11 (x32, x86) : "../lib/lz4hc.c:148: LZ4HC_countBack: Assertion `(size_t)(match - mMin) < (1U<<31)' failed." +# - all clangs (x32, x86) : "../lib/lz4hc.c:282: int LZ4HC_InsertAndGetWiderMatch(...): Assertion `matchPtr >= lowPrefixPtr' failed." +# +name: lz4 CI +on: [push, pull_request] +jobs: + lz4-c-compilers: + name: CC=${{ matrix.cc }}, ${{ matrix.os }} + strategy: + fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. + matrix: + include: [ + # You can access the following values via ${{ matrix.??? }} + # + # pkgs : apt-get package names. It can include multiple package names which are delimited by space. + # cc : C compiler executable. + # cxx : C++ compiler executable for `make ctocpptest`. + # stdc11 : Set 'true' if compiler supports C11 standard. Otherwise, set 'false'. + # stdc90 : Set 'true' if compiler supports C90 standard. Otherwise, set 'false'. + # x32 : Set 'true' if compiler supports x32. Otherwise, set 'false'. + # Set 'fail' if it supports x32 but fails for now. 'fail' cases must be removed. + # x86 : Set 'true' if compiler supports x86 (-m32). Otherwise, set 'false'. + # Set 'fail' if it supports x86 but fails for now. 'fail' cases must be removed. + # cxxtest : Set 'true' if it can be compiled as C++ code. Otherwise, set 'false'. + # os : GitHub Actions YAML workflow label. See https://github.com/actions/virtual-environments#available-environments + + # cc + { pkgs: '', cc: cc, cxx: c++, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-latest, }, + + # gcc + { pkgs: '', cc: gcc, cxx: g++, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-latest, }, + { pkgs: 'gcc-11 g++-11 lib32gcc-11-dev libx32gcc-11-dev', cc: gcc-11, cxx: g++-11, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'gcc-10 lib32gcc-10-dev libx32gcc-10-dev', cc: gcc-10, cxx: g++-10, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'gcc-9 lib32gcc-9-dev libx32gcc-9-dev', cc: gcc-9, cxx: g++-9, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'gcc-8 g++-8 lib32gcc-8-dev libx32gcc-8-dev', cc: gcc-8, cxx: g++-8, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'gcc-7 g++-7 lib32gcc-7-dev libx32gcc-7-dev', cc: gcc-7, cxx: g++-7, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'gcc-6 g++-6 lib32gcc-6-dev libx32gcc-6-dev', cc: gcc-6, cxx: g++-6, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-18.04, }, + { pkgs: 'gcc-5 g++-5 lib32gcc-5-dev libx32gcc-5-dev', cc: gcc-5, cxx: g++-5, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-18.04, }, + { pkgs: 'gcc-4.8 g++-4.8 lib32gcc-4.8-dev libx32gcc-4.8-dev', cc: gcc-4.8, cxx: g++-4.8, stdc11: 'true', stdc90: 'true', x32: 'true', x86: 'true', cxxtest: 'true', os: ubuntu-18.04, }, + { pkgs: 'gcc-4.7 g++-4.7 lib32gcc-4.7-dev', cc: gcc-4.7, cxx: g++-4.7, stdc11: 'true', stdc90: 'true', x32: 'false', x86: 'true', cxxtest: 'true', os: ubuntu-16.04, }, + { pkgs: 'gcc-4.6 g++-4.6 gcc-4.6-multilib', cc: gcc-4.6, cxx: g++-4.6, stdc11: 'false', stdc90: 'true', x32: 'false', x86: 'true', cxxtest: 'true', os: ubuntu-16.04, }, + { pkgs: 'gcc-4.4 g++-4.4 gcc-4.4-multilib', cc: gcc-4.4, cxx: g++-4.4, stdc11: 'false', stdc90: 'false', x32: 'false', x86: 'true', cxxtest: 'true', os: ubuntu-16.04, }, + + # clang + { pkgs: 'lib32gcc-11-dev libx32gcc-11-dev', cc: clang, cxx: clang++, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-latest, }, + { pkgs: 'clang-12 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-12, cxx: clang++-12, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-11 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-11, cxx: clang++-11, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-10 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-10, cxx: clang++-10, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-9 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-9, cxx: clang++-9, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-8 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-8, cxx: clang++-8, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-7 lib32gcc-7-dev libx32gcc-7-dev', cc: clang-7, cxx: clang++-7, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-6.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-6.0, cxx: clang++-6.0, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-20.04, }, + { pkgs: 'clang-5.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-5.0, cxx: clang++-5.0, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-18.04, }, + { pkgs: 'clang-4.0 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-4.0, cxx: clang++-4.0, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-18.04, }, + { pkgs: 'clang-3.9 lib32gcc-11-dev libx32gcc-11-dev', cc: clang-3.9, cxx: clang++-3.9, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'false', os: ubuntu-18.04, }, + { pkgs: 'clang-3.8', cc: clang-3.8, cxx: clang++-3.8, stdc11: 'true', stdc90: 'true', x32: 'fail', x86: 'fail', cxxtest: 'true', os: ubuntu-16.04, }, + { pkgs: 'clang-3.7', cc: clang-3.7, cxx: clang++-3.7, stdc11: 'true', stdc90: 'true', x32: 'false', x86: 'fail', cxxtest: 'true', os: ubuntu-16.04, }, + { pkgs: 'clang-3.6', cc: clang-3.6, cxx: clang++-3.6, stdc11: 'true', stdc90: 'true', x32: 'false', x86: 'fail', cxxtest: 'true', os: ubuntu-16.04, }, + { pkgs: 'clang-3.5', cc: clang-3.5, cxx: clang++-3.5, stdc11: 'true', stdc90: 'true', x32: 'false', x86: 'fail', cxxtest: 'true', os: ubuntu-16.04, }, + ] + + runs-on: ${{ matrix.os }} + env: # Set environment variables + # We globally set CC and CXX to improve compatibility with .travis.yml + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.' + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + sudo apt-get install ${{ matrix.pkgs }} + + - name: Environment info + run: | + echo && type $CC && which $CC && $CC --version + echo && type $CXX && which $CXX && $CXX --version + + - name: make + if: always() + run: make V=1 + + - name: make all + if: always() + run: make V=1 clean all + + - name: make c_standards (C90) + if: ${{ matrix.stdc90 == 'true' }} + run: make V=1 clean c_standards_c90 + + - name: make c_standards (C11) + if: ${{ matrix.stdc11 == 'true' }} + run: make V=1 clean c_standards_c11 + + - name: make c-to-c++ + if: always() + run: make V=1 clean ctocpptest + + - name: make cxxtest + if: ${{ matrix.cxxtest == 'true' }} + run: make V=1 clean cxxtest + + - name: make -C programs default + if: always() + run: make V=1 -C programs clean default + + - name: make -C programs default -D_FORTIFY_SOURCE=2 + if: always() + run: CFLAGS='-fPIC' LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make V=1 -C programs clean default + + - name: make -C tests test-lz4 + if: always() + run: MOREFLAGS='-Werror' make V=1 -C tests clean test-lz4 + + - name: make clangtest (clang only) + if: ${{ startsWith( matrix.cc , 'clang' ) }} + run: make V=1 clean clangtest + + - name: make -C tests test MOREFLAGS='-mx32' + if: ${{ matrix.x32 == 'true' }} + run: LDFLAGS='-Wl,--verbose' MOREFLAGS='-mx32' make V=1 -C tests clean test + + - name: make -C tests test-lz4c32 + if: ${{ matrix.x86 == 'true' }} + run: LDFLAGS='-Wl,--verbose' MOREFLAGS='-Werror' make V=1 -C tests clean test-lz4c32 + + + ############################################################### + # # + # Remove this block when we stabilize the tests. # + # # + + - name: make -C tests test MOREFLAGS='-mx32' || echo Ignore failure for now. + if: ${{ matrix.x32 == 'fail' }} + run: LDFLAGS='-Wl,--verbose' MOREFLAGS='-mx32' make V=1 -C tests clean test || $FIXME__LZ4_CI_IGNORE + + - name: make -C tests test-lz4c32 || echo Ignore failure for now. + if: ${{ matrix.x86 == 'fail' }} + run: LDFLAGS='-Wl,--verbose' MOREFLAGS='-Werror' make V=1 -C tests clean test-lz4c32 || $FIXME__LZ4_CI_IGNORE + + # # + ############################################################### + + + +############################################################### +# LZ4 self tests +# +# - Benchmark +# - Fuzzer +# - LZ4 Frame +# - LZ4 versions +# - Custom LZ4_DISTANCE_MAX +# + lz4-benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + + - name: benchmark (-C tests test-lz4) + run: make V=1 -C tests test-lz4 + + - name: benchmark (-C tests test-lz4c) + run: make V=1 -C tests test-lz4c + + - name: benchmark (-C tests test-lz4c32) + run: make V=1 -C tests test-lz4c32 + + - name: benchmark (-C tests test-fullbench) + run: make V=1 -C tests test-fullbench + + - name: benchmark (-C tests test-fullbench32) + run: make V=1 -C tests test-fullbench32 + + + lz4-fuzzer: + name: Fuzzer test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + + - name: setup + run: sudo sysctl -w vm.mmap_min_addr=4096 + + - name: fuzzer + run: make V=1 -C tests test-fuzzer + + - name: fuzzer32 + run: make V=1 -C tests test-fuzzer32 + + + lz4-versions: + name: LZ4 versions test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + + - name: make -C tests versionsTest + run: make V=1 -C tests versionsTest + + + lz4-frame: + name: LZ4 frame test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + + - name: LZ4 frame test + run: make V=1 -C tests test-frametest + + - name: LZ4 frame test (32-bit) + run: make V=1 -C tests test-frametest32 + + + # Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS + lz4-custom-distance: + name: Custom LZ4_DISTANCE_MAX + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: custom LZ4_DISTANCE_MAX + run: | + MOREFLAGS='-DLZ4_DISTANCE_MAX=8000' make V=1 check + make V=1 clean + make V=1 -C programs lz4-wlib + make V=1 clean + make V=1 -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS + make V=1 clean + CC="c++ -Wno-deprecated" make V=1 -C tests fullbench-wmalloc # stricter function signature check + + + +############################################################### +# Check tools +# +# - cppcheck +# - scan-build +# - valgrind +# - ubsan +# - asan +# + lz4-cppcheck: + name: make cppcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install cppcheck + + - name: Environment info + run: echo && type cppcheck && which cppcheck && cppcheck --version + + - name: cppcheck + # This test script ignores the exit code of cppcheck. + # See known issues in README.md. + run: make V=1 clean cppcheck || echo There are some cppcheck reports but we ignore it. + + + lz4-scan-build: + name: make staticAnalyze + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install clang-tools + + - name: Environment info + run: | + echo && type gcc && which gcc && gcc --version + echo && type clang && which clang && clang --version + echo && type scan-build && which scan-build # scan-build doesn't have any --version equivalent option + echo && type make && which make && make -v + echo && cat /proc/cpuinfo || echo /proc/cpuinfo is not present + + - name: make staticAnalyze + run: make V=1 clean staticAnalyze + + + lz4-valgrind: + name: valgrind + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install valgrind + + - name: Environment info + run: | + echo && type cc && which cc && cc --version + echo && type valgrind && which valgrind && valgrind --version + + - name: valgrind + run: make V=1 -C tests test-mem + + + lz4-ubsan-x64: + name: Linux x64 ubsan + runs-on: ubuntu-latest + env: # Set environment variables + FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.' + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: ubsan + ######################################################### + # For now, we ignore the exit code of `make usan`. + # See "Known issues / lz4-ubsan-x64" in README.md + # When we'll resolve this issue, remove "|| $FIXME__LZ4_CI_IGNORE" + ######################################################### + run: make V=1 clean usan MOREFLAGS='-Wcomma -Werror' || $FIXME__LZ4_CI_IGNORE + + + lz4-ubsan-x86: + name: Linux x86 ubsan + runs-on: ubuntu-latest + env: # Set environment variables + FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.' + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + sudo apt-get install lib32gcc-11-dev + + - name: ubsan32 + ######################################################### + # For now, we ignore the exit code of `make usan32`. + # See "Known issues / lz4-ubsaan-x86" in README.md. + # When we'll resolve this issue, remove "|| $FIXME__LZ4_CI_IGNORE" + ######################################################### + run: CC=clang make V=1 clean usan32 MOREFLAGS='-Wcomma -Werror' || $FIXME__LZ4_CI_IGNORE + + + lz4-asan-x64: + name: Linux x64 ASAN + runs-on: ubuntu-latest + env: # Set environment variables + FIXME__LZ4_CI_IGNORE : ' echo Error. But we ignore it for now.' + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: setup + run: sudo sysctl -w vm.mmap_min_addr=4096 + + - name: frametest + run: CC=clang MOREFLAGS=-fsanitize=address make V=1 -C tests clean test-frametest + + - name: fuzzer + run: CC=clang MOREFLAGS=-fsanitize=address make V=1 -C tests clean test-fuzzer + + + +############################################################### +# Platforms +# +# - QEMU (ARM, ARM64, PPC, PPC64LE, S390X) +# - macOS +# + + # QEMU + # All tests use QEMU (static) and gcc cross compiler. + # + # note: + # We don't employ completely matrix method which provides `MOREFLAGS` + # etc in the matrix. Because some platform may need its special + # compiler options and test. + # For example, xxHash already has tests for scalar and SIMD version of + # it. But compiler options are quite different between platforms. + # + # So, please keep them simple and independent. + # + lz4-qemu-platforms: + name: QEMU ${{ matrix.type }} + strategy: + fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix instance failed. + matrix: + include: [ + # You can access the following values via ${{ matrix.??? }} + # type : Architecture type for `if:` statement. + # pkgs : apt-get package names. You can include multiple packages which are delimited by space. + # xcc : gcc cross C compiler executable. + # xemu : QEMU static emulator executable. + # os : GitHub Actions YAML workflow label. See https://github.com/actions/virtual-environments#available-environments + + { type: ARM, pkgs: 'qemu-system-arm gcc-arm-linux-gnueabi', xcc: arm-linux-gnueabi-gcc, xemu: qemu-arm-static, os: ubuntu-latest, }, + { type: ARM64, pkgs: 'qemu-system-arm gcc-aarch64-linux-gnu', xcc: aarch64-linux-gnu-gcc, xemu: qemu-aarch64-static, os: ubuntu-latest, }, + { type: PPC, pkgs: 'qemu-system-ppc gcc-powerpc-linux-gnu', xcc: powerpc-linux-gnu-gcc, xemu: qemu-ppc-static, os: ubuntu-latest, }, + { type: PPC64LE, pkgs: 'qemu-system-ppc gcc-powerpc64le-linux-gnu', xcc: powerpc64le-linux-gnu-gcc, xemu: qemu-ppc64le-static, os: ubuntu-latest, }, + { type: S390X, pkgs: 'qemu-system-s390x gcc-s390x-linux-gnu', xcc: s390x-linux-gnu-gcc, xemu: qemu-s390x-static, os: ubuntu-latest, }, + ] + + runs-on: ${{ matrix.os }} + env: # Set environment variables + XCC: ${{ matrix.xcc }} + XEMU: ${{ matrix.xemu }} + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: apt-get install + run: | + sudo apt-get update + sudo apt-get install gcc-multilib + sudo apt-get install qemu-utils qemu-user-static + sudo apt-get install ${{ matrix.pkgs }} + + - name: Environment info + run: | + echo && type $XCC && which $XCC && $XCC --version + echo && $XCC -v # Show built-in specs + echo && type $XEMU && which $XEMU && $XEMU --version + + - name: ARM64 + if: ${{ matrix.type == 'ARM64' }} + run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU + + - name: ARM + if: ${{ matrix.type == 'ARM' }} + run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU + + - name: PPC + if: ${{ matrix.type == 'PPC' }} + run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU + + - name: PPC64LE + if: ${{ matrix.type == 'PPC64LE' }} + run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU MOREFLAGS=-m64 + + - name: S390X + if: ${{ matrix.type == 'S390X' }} + run: make V=1 platformTest CC=$XCC QEMU_SYS=$XEMU + + + # macOS + lz4-platform-macos-latest: + name: macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + + - name: Environment info + run: | + echo && type cc && which cc && cc --version + echo && type make && which make && make -v + echo && sysctl -a | grep machdep.cpu # cpuinfo + + - name: make default + run: CFLAGS="-Werror" make V=1 clean default + + - name: make test + run: make V=1 clean test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' + + - name: make test | tee + # test scenario where `stdout` is not the console + run: make V=1 clean test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee + + + +############################################################### +# Build systems +# +# - make +# - cmake +# - meson +# + + # make + lz4-build-make: + name: make + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: Environment info + run: | + echo && type cc && which cc && cc --version + echo && type make && which make && make -v + + - name: make + run: make V=1 + + + lz4-build-make-travis-install: + name: make travis-install + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: travis-install + run: make V=1 clean travis-install + + - name: travis-install result + run: | + echo && echo Installed files + ( cd ~/install_test_dir; find .; ) + + + # cmake + lz4-build-cmake: + name: cmake + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + + - name: Environment info + run: | + echo && type cmake && which cmake && cmake --version + echo && type make && which make && make -v + + - name: cmake + run: | + cd build/cmake + mkdir build + cd build + cmake .. + CFLAGS=-Werror make VERBOSE=1 + + + # Invoke cmake via Makefile + lz4-build-make-cmake: + name: make cmake + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - name: make cmake + # V=1 for lz4 Makefile, VERBOSE=1 for cmake Makefile. + run: make V=1 VERBOSE=1 clean cmake + + + # Meson + lz4-build-meson: + name: Meson + Ninja + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 # https://github.com/actions/checkout + - uses: actions/setup-python@v2 # https://github.com/actions/setup-python + with: + python-version: '3.x' + + - name: Install + run: | + sudo apt-get update + sudo apt-get install tree ninja-build + python -m pip install --upgrade pip + pip3 install --user meson + + - name: Environment info + run: | + echo && type clang && which clang && clang --version + echo && type python && which python && python --version + echo && type meson && which meson && meson --version + + - name: meson + # 'run: >' replaces all newlines in the following block with spaces + run: > + meson setup + --buildtype=debug + -Db_lundef=false + -Dauto_features=enabled + -Ddefault_library=both + -Dbin_programs=true + -Dbin_contrib=true + -Dbin_tests=true + -Dbin_examples=true + contrib/meson build + + - name: staging + run: | + cd build + DESTDIR=./staging ninja install + tree ./staging + + + +############################################################ +# Check git tag for LZ4 releases +# + lz4-check-tag: + name: git version tag checking for release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make -C tests checkTag + if: startsWith(github.ref, 'refs/tags/v') # If git tag name starts with 'v' + run: | + echo "tag=${GITHUB_REF#refs/*/}" + make -C tests checkTag + tests/checkTag ${GITHUB_REF#refs/*/} + + + +############################################################ +# Gather CI environment information. +# + lz4-env-info: + name: GH-Actions Virtual Env Info (${{ matrix.os }}) + strategy: + matrix: + include: [ + { os: ubuntu-latest, }, # https://github.com/actions/virtual-environments/ + { os: ubuntu-20.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md + { os: ubuntu-18.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md + { os: ubuntu-16.04, }, # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1604-README.md + ] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + - name: init + run: | + sudo apt-get update + + - name: cc --version + run: echo && type cc && which cc && cc --version + + - name: gcc --version + run: echo && type gcc && which gcc && gcc --version + + - name: clang --version + run: echo && type clang && which clang && clang --version + + - name: make -v + run: echo && type make && which make && make -v + + - name: g++ --version + run: echo && type g++ && which g++ && g++ --version + + - name: git --version + run: echo && type git && which git && git --version + + - name: gcc packages (apt-cache) + run: apt-cache search gcc | grep "^gcc-[0-9\.]* " | sort + + - name: lib32gcc packages for i386 (apt-cache) + run: apt-cache search lib32gcc | grep "^lib32gcc-" | sort + + - name: libx32gcc packages for x32 (apt-cache) + run: apt-cache search libx32gcc | grep "^libx32gcc-" | sort + + - name: gcc multilib packages (apt-cache) + run: apt-cache search multilib | grep "gcc-" | sort + + - name: clang packages (apt-cache) + run: apt-cache search clang | grep "^clang-[0-9\.]* " | sort + + - name: QEMU packages (apt-cache) + run: apt-cache search qemu | grep "^qemu-system-.*QEMU full system" | sort diff --git a/.travis.yml b/.travis.yml index bdab59f..0616021 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,126 +3,126 @@ language: c matrix: fast_finish: true include: - # OS X Mavericks - - name: (macOS) General Test - os: osx - compiler: clang - script: - - make # test library build - - make clean - - make test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console - - # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - name: (Precise) benchmark test - dist: precise - script: - - make -C tests test-lz4 test-lz4c test-fullbench - - - name: (Precise) frame and fuzzer test - dist: precise - install: - - sudo sysctl -w vm.mmap_min_addr=4096 - script: - - make -C tests test-frametest test-fuzzer - - - name: ASAN tests with fuzzer and frametest - install: - - sudo sysctl -w vm.mmap_min_addr=4096 - script: - - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer - - - name: Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS - script: - - MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check - - make clean - - make -C programs lz4-wlib - - make clean - - make -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS - - make clean - - CC="c++ -Wno-deprecated" make -C tests fullbench-wmalloc # stricter function signature check - - - name: (Precise) g++ and clang CMake test - dist: precise - script: - - make cxxtest - - make clean - - make examples - - make clean cmake - - make clean travis-install - - make clean clangtest - - - name: x32 compatibility test - addons: - apt: - packages: - - gcc-multilib - script: - - make -C tests test MOREFLAGS=-mx32 - - # 14.04 LTS Server Edition 64 bit - # presume clang >= v3.9.0 - - name: (Trusty) USan test - dist: trusty - compiler: clang - script: - - make usan MOREFLAGS=-Wcomma -Werror - - - name: (Trusty) valgrind test - dist: trusty - install: - - sudo apt-get install -qq valgrind - script: - - make c_standards - - make -C tests test-lz4 test-mem - - - name: (Trusty) c-to-c++ test - dist: trusty - script: - - make ctocpptest - - - name: (Trusty) i386 benchmark + version test - dist: trusty - install: - - sudo apt-get install -qq python3 libc6-dev-i386 gcc-multilib - script: - - make -C tests test-lz4c32 test-fullbench32 versionsTest - - - name: (Trusty) i386 frame + fuzzer test - dist: trusty - install: - - sudo apt-get install -qq libc6-dev-i386 gcc-multilib - - sudo sysctl -w vm.mmap_min_addr=4096 - script: - - make -C tests test-frametest32 test-fuzzer32 - - - name: (Trusty) gcc-6 standard C compilation - dist: trusty - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-6 - env: - - CC=gcc-6 - script: - - make c_standards - - make -C tests test-lz4 MOREFLAGS=-Werror - - - name: (Trusty) arm + aarch64 compilation - dist: trusty - install: - - sudo apt-get install -qq - qemu-system-arm - qemu-user-static - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - gcc-aarch64-linux-gnu - libc6-dev-arm64-cross - script: - - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static - - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static - +# # OS X Mavericks +# - name: (macOS) General Test +# os: osx +# compiler: clang +# script: +# - make # test library build +# - make clean +# - make test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console +# +# # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) +# - name: (Precise) benchmark test +# dist: precise +# script: +# - make -C tests test-lz4 test-lz4c test-fullbench +# +# - name: (Precise) frame and fuzzer test +# dist: precise +# install: +# - sudo sysctl -w vm.mmap_min_addr=4096 +# script: +# - make -C tests test-frametest test-fuzzer +# +# - name: ASAN tests with fuzzer and frametest +# install: +# - sudo sysctl -w vm.mmap_min_addr=4096 +# script: +# - CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer +# +# - name: Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS +# script: +# - MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check +# - make clean +# - make -C programs lz4-wlib +# - make clean +# - make -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS +# - make clean +# - CC="c++ -Wno-deprecated" make -C tests fullbench-wmalloc # stricter function signature check +# +# - name: (Precise) g++ and clang CMake test +# dist: precise +# script: +# - make cxxtest +# - make clean +# - make examples +# - make clean cmake +# - make clean travis-install +# - make clean clangtest +# +# - name: x32 compatibility test +# addons: +# apt: +# packages: +# - gcc-multilib +# script: +# - make -C tests test MOREFLAGS=-mx32 +# +# # 14.04 LTS Server Edition 64 bit +# # presume clang >= v3.9.0 +# - name: (Trusty) USan test +# dist: trusty +# compiler: clang +# script: +# - make usan MOREFLAGS=-Wcomma -Werror +# +# - name: (Trusty) valgrind test +# dist: trusty +# install: +# - sudo apt-get install -qq valgrind +# script: +# - make c_standards +# - make -C tests test-lz4 test-mem +# +# - name: (Trusty) c-to-c++ test +# dist: trusty +# script: +# - make ctocpptest +# +# - name: (Trusty) i386 benchmark + version test +# dist: trusty +# install: +# - sudo apt-get install -qq python3 libc6-dev-i386 gcc-multilib +# script: +# - make -C tests test-lz4c32 test-fullbench32 versionsTest +# +# - name: (Trusty) i386 frame + fuzzer test +# dist: trusty +# install: +# - sudo apt-get install -qq libc6-dev-i386 gcc-multilib +# - sudo sysctl -w vm.mmap_min_addr=4096 +# script: +# - make -C tests test-frametest32 test-fuzzer32 +# +# - name: (Trusty) gcc-6 standard C compilation +# dist: trusty +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# packages: +# - gcc-6 +# env: +# - CC=gcc-6 +# script: +# - make c_standards +# - make -C tests test-lz4 MOREFLAGS=-Werror +# +# - name: (Trusty) arm + aarch64 compilation +# dist: trusty +# install: +# - sudo apt-get install -qq +# qemu-system-arm +# qemu-user-static +# gcc-arm-linux-gnueabi +# libc6-dev-armel-cross +# gcc-aarch64-linux-gnu +# libc6-dev-arm64-cross +# script: +# - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static +# - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static +# - name: aarch64 real-hw tests arch: arm64 script: @@ -138,56 +138,56 @@ matrix: script: - make test - - name: (Xenial) gcc-5 compilation - dist: xenial - install: - - sudo apt-get install -qq libc6-dev-i386 gcc-multilib - script: - - make -C tests test-lz4 clean test-lz4c32 MOREFLAGS=-Werror - - - name: (Trusty) clang-3.8 compilation - dist: trusty - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - packages: - - clang-3.8 - script: - - make -C tests test-lz4 CC=clang-3.8 - - - name: (Trusty) PowerPC + PPC64 compilation - dist: trusty - install: - - sudo apt-get install -qq qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu - script: - - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static - - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 - - - name: (Trusty) scan-build + cppcheck - dist: trusty - compiler: clang - install: - - sudo apt-get install -qq cppcheck - script: - - make staticAnalyze - - make cppcheck - - - name: (Trusty) gcc-4.4 compilation - dist: trusty - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-4.4 - script: - - make clean all CC=gcc-4.4 MOREFLAGS=-Werror - - make clean - - CFLAGS=-fPIC LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make -C programs +# - name: (Xenial) gcc-5 compilation +# dist: xenial +# install: +# - sudo apt-get install -qq libc6-dev-i386 gcc-multilib +# script: +# - make -C tests test-lz4 clean test-lz4c32 MOREFLAGS=-Werror +# +# - name: (Trusty) clang-3.8 compilation +# dist: trusty +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# - llvm-toolchain-precise-3.8 +# packages: +# - clang-3.8 +# script: +# - make -C tests test-lz4 CC=clang-3.8 +# +# - name: (Trusty) PowerPC + PPC64 compilation +# dist: trusty +# install: +# - sudo apt-get install -qq qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu +# script: +# - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static +# - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 +# +# - name: (Trusty) scan-build + cppcheck +# dist: trusty +# compiler: clang +# install: +# - sudo apt-get install -qq cppcheck +# script: +# - make staticAnalyze +# - make cppcheck +# +# - name: (Trusty) gcc-4.4 compilation +# dist: trusty +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# packages: +# - libc6-dev-i386 +# - gcc-multilib +# - gcc-4.4 +# script: +# - make clean all CC=gcc-4.4 MOREFLAGS=-Werror +# - make clean +# - CFLAGS=-fPIC LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make -C programs # tag-specific test - name: tag build @@ -197,35 +197,35 @@ matrix: - make -C tests checkTag - tests/checkTag "$TRAVIS_BRANCH" - - name: (Xenial) Meson + clang build - #env: ALLOW_FAILURES=true - dist: xenial - language: cpp - compiler: clang - install: - - sudo apt-get install -qq python3 tree - - | - travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' && - unzip ~/ninja.zip -d ~/.local/bin - - | - travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && - python3 ~/get-pip.py --user && - pip3 install --user meson - script: - - | - meson setup \ - --buildtype=debug \ - -Db_lundef=false \ - -Dauto_features=enabled \ - -Ddefault_library=both \ - -Dbin_programs=true \ - -Dbin_contrib=true \ - -Dbin_tests=true \ - -Dbin_examples=true \ - contrib/meson build - - pushd build - - DESTDIR=./staging ninja install - - tree ./staging +# - name: (Xenial) Meson + clang build +# #env: ALLOW_FAILURES=true +# dist: xenial +# language: cpp +# compiler: clang +# install: +# - sudo apt-get install -qq python3 tree +# - | +# travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' && +# unzip ~/ninja.zip -d ~/.local/bin +# - | +# travis_retry curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && +# python3 ~/get-pip.py --user && +# pip3 install --user meson +# script: +# - | +# meson setup \ +# --buildtype=debug \ +# -Db_lundef=false \ +# -Dauto_features=enabled \ +# -Ddefault_library=both \ +# -Dbin_programs=true \ +# -Dbin_contrib=true \ +# -Dbin_tests=true \ +# -Dbin_examples=true \ +# contrib/meson build +# - pushd build +# - DESTDIR=./staging ninja install +# - tree ./staging # oss-fuzz compilation test - name: Compile OSS-Fuzz targets @@ -155,7 +155,7 @@ usan: CC = clang usan: CFLAGS = -O3 -g -fsanitize=undefined -fno-sanitize-recover=undefined -fsanitize-recover=pointer-overflow usan: LDFLAGS = $(CFLAGS) usan: clean - $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 + CC=$(CC) CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1 .PHONY: usan32 usan32: CFLAGS = -m32 -O3 -g -fsanitize=undefined @@ -215,11 +215,20 @@ ctocpptest: clean CC=$(TESTCC) $(MAKE) -C $(TESTDIR) CFLAGS="$(CFLAGS)" all .PHONY: c_standards -c_standards: clean +c_standards: clean c_standards_c11 c_standards_c99 c_standards_c90 + +.PHONY: c_standards_c90 +c_standards_c90: clean $(MAKE) clean; CFLAGS="-std=c90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost $(MAKE) clean; CFLAGS="-std=gnu90 -Werror -pedantic -Wno-long-long -Wno-variadic-macros" $(MAKE) allmost + +.PHONY: c_standards_c99 +c_standards_c99: clean $(MAKE) clean; CFLAGS="-std=c99 -Werror -pedantic" $(MAKE) all $(MAKE) clean; CFLAGS="-std=gnu99 -Werror -pedantic" $(MAKE) all + +.PHONY: c_standards_c11 +c_standards_c11: clean $(MAKE) clean; CFLAGS="-std=c11 -Werror" $(MAKE) all endif # MSYS POSIX diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index 19c3345..3047a3a 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -65,7 +65,7 @@ static void test_compress( { const int cmpBytes = LZ4_compress_fast_continue( - lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1); + lz4Stream, inpPtr, cmpBuf, inpBytes, (int) cmpBufBytes, 1); if (cmpBytes <= 0) break; write_uint16(outFp, (uint16_t) cmpBytes); write_bin(outFp, cmpBuf, cmpBytes); diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c index ecb3b2d..f7d1b67 100644 --- a/examples/dictionaryRandomAccess.c +++ b/examples/dictionaryRandomAccess.c @@ -97,7 +97,7 @@ void test_compress(FILE* outFp, FILE* inpFp, void *dict, int dictSize) while (ptr != offsetsEnd) { write_int(outFp, *ptr++); } - write_int(outFp, offsetsEnd - offsets); + write_int(outFp, (int) (offsetsEnd - offsets)); } } @@ -652,10 +652,10 @@ typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere * else in memory, starting at ctx->dictionary with length * ctx->dictSize. - * - usingDictCtx : Like usingExtDict, but everything concerning the preceding - * content is in a separate context, pointed to by - * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table - * entries in the current context that refer to positions + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions * preceding the beginning of the current compression are * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx * ->dictSize describe the location and size of the preceding @@ -675,9 +675,9 @@ int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; } -/*-************************************ -* Internal Definitions used in Tests -**************************************/ +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ #if defined (__cplusplus) extern "C" { #endif @@ -827,9 +827,10 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx, } } - /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster - * than compressing without a gap. However, compressing with - * currentOffset == 0 is faster still, so we preserve that case. + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. */ if (cctx->currentOffset != 0 && tableType == byU32) { DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); @@ -885,7 +886,8 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( /* the dictCtx currentOffset is indexed on the start of the dictionary, * while a dictionary in the current context precedes the currentOffset */ - const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ? + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? dictionary + dictSize - dictCtx->currentOffset : dictionary + dictSize - startIndex; @@ -981,10 +983,11 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated( match = base + matchIndex; lowLimit = (const BYTE*)source; } - } else if (dictDirective==usingExtDict) { + } else if (dictDirective == usingExtDict) { if (matchIndex < startIndex) { DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); match = dictBase + matchIndex; lowLimit = dictionary; } else { @@ -1173,6 +1176,7 @@ _next_match: } } else if (dictDirective==usingExtDict) { if (matchIndex < startIndex) { + assert(dictBase); match = dictBase + matchIndex; lowLimit = dictionary; /* required for match length counter */ } else { @@ -1355,7 +1359,7 @@ int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutp { int result; #if (LZ4_HEAPMODE) - LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ if (ctxPtr == NULL) return 0; #else LZ4_stream_t ctx; @@ -1514,8 +1518,9 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) return (int)dict->dictSize; } -void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { - const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : &(dictionaryStream->internal_donotuse); DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", @@ -1568,36 +1573,40 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, int acceleration) { const tableType_t tableType = byU32; - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; - DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); - LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; /* invalidate tiny dictionaries */ - if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ - && (dictEnd != (const BYTE*)source) ) { + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ streamPtr->dictSize = 0; streamPtr->dictionary = (const BYTE*)source; - dictEnd = (const BYTE*)source; + dictEnd = source; } /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; } } /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { + if (dictEnd == source) { if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else @@ -1623,7 +1632,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, } else { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); } - } else { + } else { /* small data <= 4 KB */ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); } else { @@ -1661,21 +1670,25 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* /*! LZ4_saveDict() : * If previously compressed data block is not guaranteed to remain available at its memory location, * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. */ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } if (safeBuffer == NULL) assert(dictSize == 0); - if (dictSize > 0) + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + } dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -2285,8 +2298,13 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } lz4sd->externalDict = NULL; lz4sd->extDictSize = 0; return 1; diff --git a/lib/lz4frame.c b/lib/lz4frame.c index ec02c92..21dc47f 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -228,9 +228,9 @@ typedef struct LZ4F_cctx_s const LZ4F_CDict* cdict; size_t maxBlockSize; size_t maxBufferSize; - BYTE* tmpBuff; - BYTE* tmpIn; - size_t tmpInSize; + BYTE* tmpBuff; /* internal buffer, for streaming */ + BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */ + size_t tmpInSize; /* amount of data to compress after tmpIn */ U64 totalInSize; XXH32_state_t xxh; void* lz4CtxPtr; @@ -539,7 +539,7 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionConte if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed); cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : init stream */ + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ *LZ4F_compressionContextPtr = cctxPtr; @@ -590,9 +590,9 @@ static void LZ4F_initStream(void* ctx, /*! LZ4F_compressBegin_usingCDict() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * @return : number of bytes written into dstBuffer for the header + * init streaming compression AND writes frame header into @dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @return : number of bytes written into @dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, @@ -600,19 +600,18 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, const LZ4F_CDict* cdict, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefNull; + LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; - BYTE* headerStart; if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - MEM_INIT(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; - /* Ctx Management */ + /* cctx Management */ { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; if (cctxPtr->lz4CtxAlloc < ctxTypeID) { + /* not enough space allocated */ FREEMEM(cctxPtr->lz4CtxPtr); if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { cctxPtr->lz4CtxPtr = LZ4_createStream(); @@ -624,13 +623,13 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, cctxPtr->lz4CtxAlloc = ctxTypeID; cctxPtr->lz4CtxState = ctxTypeID; } else if (cctxPtr->lz4CtxState != ctxTypeID) { - /* otherwise, a sufficient buffer is allocated, but we need to - * reset it to the correct context type */ + /* otherwise, a sufficient buffer is already allocated, + * but we need to reset it to the correct context type */ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { - LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t)); + LZ4_initStream((LZ4_stream_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_stream_t)); } else { - LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); - LZ4_setCompressionLevel((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + LZ4_initStreamHC((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); } cctxPtr->lz4CtxState = ctxTypeID; } @@ -669,31 +668,32 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, /* Magic Number */ LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); dstPtr += 4; - headerStart = dstPtr; - - /* FLG Byte */ - *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) - + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) - + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) - + (cctxPtr->prefs.frameInfo.dictID > 0) ); - /* BD Byte */ - *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - /* Optional Frame content size field */ - if (cctxPtr->prefs.frameInfo.contentSize) { - LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); - dstPtr += 8; - cctxPtr->totalInSize = 0; - } - /* Optional dictionary ID field */ - if (cctxPtr->prefs.frameInfo.dictID) { - LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); - dstPtr += 4; + { BYTE* const headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((unsigned)(cctxPtr->prefs.frameInfo.contentSize > 0) << 3) + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctxPtr->prefs.frameInfo.dictID > 0) ); + /* BD Byte */ + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.contentSize) { + LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } + /* Optional dictionary ID field */ + if (cctxPtr->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); + dstPtr++; } - /* Header CRC Byte */ - *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); - dstPtr++; cctxPtr->cStage = 1; /* header written, now request input data block */ return (size_t)(dstPtr - dstStart); @@ -701,9 +701,9 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, /*! LZ4F_compressBegin() : - * init streaming compression and writes frame header into dstBuffer. - * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes. - * preferencesPtr can be NULL, in which case default parameters are selected. + * init streaming compression AND writes frame header into @dstBuffer. + * @dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * @preferencesPtr can be NULL, in which case default parameters are selected. * @return : number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) */ @@ -766,6 +766,7 @@ static size_t LZ4F_makeBlock(void* dst, static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { int const acceleration = (level < 0) ? -level + 1 : 1; + DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize); LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); if (cdict) { return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); @@ -778,6 +779,7 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in { int const acceleration = (level < 0) ? -level + 1 : 1; (void)cdict; /* init once at beginning of frame */ + DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize); return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); } @@ -806,6 +808,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev return LZ4F_compressBlockHC_continue; } +/* Save history (up to 64KB) into @tmpBuff */ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) @@ -815,19 +818,23 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; +static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; + /*! LZ4F_compressUpdate() : * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). - * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. */ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull; size_t const blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; @@ -838,15 +845,15 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); - if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); + if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); /* state must be initialized and waiting for next block */ if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); - MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull)); - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; /* complete tmp buffer */ if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; + assert(blockSize > cctxPtr->tmpInSize); if (sizeToCopy > srcSize) { /* add src to tmpIn buffer */ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); @@ -867,8 +874,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; - } - } + } } while ((size_t)(srcEnd - srcPtr) >= blockSize) { /* compress full blocks */ @@ -882,7 +888,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, } if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { - /* compress remaining input < blockSize */ + /* autoFlush : remaining input (< blockSize) is compressed */ lastBlockCompressed = fromSrcBuffer; dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, (size_t)(srcEnd - srcPtr), @@ -892,10 +898,10 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, srcPtr = srcEnd; } - /* preserve dictionary if necessary */ + /* preserve dictionary within @tmpBuff whenever necessary */ if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { if (compressOptionsPtr->stableSrc) { - cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ } else { int const realDictSize = LZ4F_localSaveDict(cctxPtr); if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC); @@ -904,11 +910,14 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, } /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ - && !(cctxPtr->prefs.autoFlush)) + if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */ + && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */ { + /* only preserve 64KB within internal buffer. Ensures there is enough room for next block. + * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */ int const realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)); } /* some input data left, necessarily < blockSize */ diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 267f289..9ac849f 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -248,7 +248,8 @@ LZ4FLIB_API unsigned LZ4F_getVersion(void); * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. * The function will provide a pointer to a fully allocated LZ4F_cctx object. * If @return != zero, there was an error during context creation. - * Object can release its memory using LZ4F_freeCompressionContext(); + * Object can be released using LZ4F_freeCompressionContext(); + * Note: LZ4F_freeCompressionContext() works with NULL pointers (do nothing). */ LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); @@ -301,8 +302,7 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. * This value is provided by LZ4F_compressBound(). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - * LZ4F_compressUpdate() doesn't guarantee error recovery. - * When an error occurs, compression context must be freed or resized. + * After an error, the state is left in a UB state, and must be re-initialized or freed. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * or an error code if it fails (which can be tested using LZ4F_isError()) diff --git a/tests/checkFrame.c b/tests/checkFrame.c index 2b50bae..946805f 100644 --- a/tests/checkFrame.c +++ b/tests/checkFrame.c @@ -153,7 +153,7 @@ int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSiz if (LZ4F_isError(nextToLoad)) EXM_THROW(22, "Error getting frame info: %s", LZ4F_getErrorName(nextToLoad)); - if (frameInfo.blockSizeID != bsid) + if (frameInfo.blockSizeID != (LZ4F_blockSizeID_t) bsid) EXM_THROW(23, "Block size ID %u != expected %u", frameInfo.blockSizeID, bsid); pos += remaining; diff --git a/tests/frametest.c b/tests/frametest.c index f06d9b7..b3b4df8 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -558,7 +558,10 @@ int basicTests(U32 seed, double compressibility) cdict, NULL) ); DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", (unsigned)dictSize, (unsigned)cSizeWithDict); - if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */ + if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) { + DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict); + goto _output_error; /* must be more efficient */ + } crcOrig = XXH64(CNBuffer, dictSize, 0); DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); |