diff options
-rw-r--r-- | .travis.yml | 245 | ||||
-rw-r--r-- | contrib/meson/GetLz4LibraryVersion.py | 20 | ||||
-rw-r--r-- | contrib/meson/InstallSymlink.py | 65 | ||||
-rw-r--r-- | contrib/meson/README.md | 34 | ||||
-rw-r--r-- | contrib/meson/lib/meson.build | 15 | ||||
-rw-r--r-- | contrib/meson/meson.build | 63 | ||||
-rw-r--r-- | contrib/meson/programs/meson.build | 5 | ||||
-rw-r--r-- | contrib/meson/tests/meson.build | 20 | ||||
-rw-r--r-- | contrib/snap/README.md | 29 | ||||
-rw-r--r-- | contrib/snap/snapcraft.yaml | 31 | ||||
-rw-r--r-- | doc/lz4_Block_format.md | 69 | ||||
-rw-r--r-- | doc/lz4_Frame_format.md | 17 | ||||
-rw-r--r-- | lib/lz4.c | 268 | ||||
-rw-r--r-- | lib/lz4frame.c | 15 | ||||
-rw-r--r-- | lib/lz4frame.h | 1 | ||||
-rw-r--r-- | lib/lz4hc.c | 20 | ||||
-rw-r--r-- | programs/lz4cli.c | 93 | ||||
-rw-r--r-- | programs/lz4io.c | 269 | ||||
-rw-r--r-- | programs/lz4io.h | 44 | ||||
-rw-r--r-- | tests/.gitignore | 1 | ||||
-rw-r--r-- | tests/Makefile | 14 | ||||
-rw-r--r-- | tests/frametest.c | 23 | ||||
-rw-r--r-- | tests/fuzzer.c | 118 | ||||
-rw-r--r-- | visual/VS2017/lz4.sln | 17 |
24 files changed, 974 insertions, 522 deletions
diff --git a/.travis.yml b/.travis.yml index 043f932..301d294 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,116 +1,123 @@ language: c + matrix: fast_finish: true include: # OS X Mavericks - - os: osx - install: - - export CC=clang - env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lz4 MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" && CFLAGS=-m32 make -C tests clean test-lz4-contentSize' COMPILER=clang + - name: (macOS) General Test + os: osx + compiler: clang + script: + - make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' + - CFLAGS=-m32 make -C tests clean test-lz4-contentSize # Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - os: linux - sudo: false - env: Ubu=12.04cont Cmd='make -C tests test-lz4 test-lz4c test-fullbench' COMPILER=cc + - name: (Precise) benchmark test + dist: precise + script: + - make -C tests test-lz4 test-lz4c test-fullbench - - os: linux - sudo: required - env: Ubu=12.04cont Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest test-fuzzer' COMPILER=cc + - name: (Precise) frame and fuzzer test + dist: precise + install: + - sudo sysctl -w vm.mmap_min_addr=4096 + script: + - make -C tests test-frametest test-fuzzer - - os: linux - sudo: false - env: Ubu=12.04cont Cmd="make gpptest && make clean && make examples && make clean cmake && make clean travis-install && make clean clangtest" COMPILER=cc + - name: (Precise) g++ and clang CMake test + dist: precise + script: + - make gpptest + - make clean + - make examples + - make clean cmake + - make clean travis-install + - make clean clangtest # 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make -C tests test MOREFLAGS=-mx32' COMPILER=cc + - name: (Trusty) i386 gcc test dist: trusty - sudo: required addons: apt: packages: - libc6-dev-i386 - gcc-multilib + script: + - make -C tests test MOREFLAGS=-mx32 # presume clang >= v3.9.0 - - env: Ubu=14.04 Cmd='make usan MOREFLAGS=-Wcomma -Werror' COMPILER=clang + - name: (Trusty) USan test dist: trusty - sudo: required - addons: - apt: - packages: - - clang + compiler: clang + script: + - make usan MOREFLAGS=-Wcomma -Werror - - env: Ubu=14.04 Cmd='make c_standards && make -C tests test-lz4 test-mem' COMPILER=cc + - name: (Trusty) valgrind test dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind + install: + - sudo apt-get install -qq valgrind + script: + - make c_standards + - make -C tests test-lz4 test-mem - - env: Ubu=14.04 Cmd='make ctocpptest' COMPILER=cc + - name: (Trusty) c-to-c++ test dist: trusty - sudo: false + script: + - make ctocpptest - - env: Ubu=14.04 Cmd='make -C tests test-lz4c32 test-fullbench32 versionsTest' COMPILER=cc + - name: (Trusty) i386 benchmark + version test dist: trusty - sudo: required - addons: - apt: - packages: - - python3 - - libc6-dev-i386 - - gcc-multilib + install: + - sudo apt-get install -qq python3 libc6-dev-i386 gcc-multilib + script: + - make -C tests test-lz4c32 test-fullbench32 versionsTest - - env: Ubu=14.04 Cmd='sudo sysctl -w vm.mmap_min_addr="4096" && make -C tests test-frametest32 test-fuzzer32' COMPILER=cc + - name: (Trusty) i386 frame + fuzzer test dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib + install: + - sudo apt-get install -qq libc6-dev-i386 gcc-multilib + - sudo sysctl -w vm.mmap_min_addr=4096 + script: + - make -C tests test-frametest32 test-fuzzer32 - - env: Ubu=14.04 Cmd='make c_standards CC=gcc-6 && make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror' COMPILER=gcc-6 + - name: (Trusty) gcc-6 standard C compilation dist: trusty - sudo: required addons: apt: sources: - ubuntu-toolchain-r-test packages: - gcc-6 + env: + - CC=gcc-6 + script: + - make c_standards + - make -C tests test-lz4 MOREFLAGS=-Werror - - env: Ubu=14.04 Cmd='make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static' COMPILER=arm-linux-gnueabi-gcc + - name: (Trusty) arm + aarch64 compilation dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross - - gcc-aarch64-linux-gnu - - libc6-dev-arm64-cross - - - env: Ubu=14.04 Cmd='make -C tests test-lz4 clean test-lz4c32 CC=gcc-5 MOREFLAGS=-Werror' COMPILER=gcc-5 - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-5 - - gcc-5-multilib + install: + - sudo apt-get install -qq + qemu-system-arm + qemu-user-static + gcc-arm-linux-gnueabi + libc6-dev-armel-cross + gcc-aarch64-linux-gnu + libc6-dev-arm64-cross + script: + - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static + - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static + + - name: (Xenial) gcc-5 compilation + dist: xenial + install: + - sudo apt-get install -qq libc6-dev-i386 gcc-multilib + script: + - make -C tests test-lz4 clean test-lz4c32 MOREFLAGS=-Werror - - env: Ubu=14.04 Cmd='make -C tests test-lz4 CC=clang-3.8' COMPILER=clang-3.8 + - name: (Trusty) clang-3.8 compilation dist: trusty - sudo: required addons: apt: sources: @@ -118,29 +125,28 @@ matrix: - llvm-toolchain-precise-3.8 packages: - clang-3.8 + script: + - make -C tests test-lz4 CC=clang-3.8 - - env: Ubu=14.04 Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc + - name: (Trusty) PowerPC + PPC64 compilation dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu + install: + - sudo apt-get install -qq qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu + script: + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 - - env: Ubu=14.04 Cmd='make staticAnalyze && make cppcheck' COMPILER=clang + - name: (Trusty) scan-build + cppcheck dist: trusty - sudo: required - addons: - apt: - packages: - - clang - - cppcheck + compiler: clang + install: + - sudo apt-get install -qq cppcheck + script: + - make staticAnalyze + - make cppcheck - - env: Ubu=14.04 Cmd='make clean all CC=gcc-4.4 MOREFLAGS=-Werror && make clean && CFLAGS=-fPIC LDFLAGS="-pie -fPIE -D_FORTIFY_SOURCE=2" make -C programs' COMPILER=gcc-4.4 + - name: (Trusty) gcc-4.4 compilation dist: trusty - sudo: required addons: apt: sources: @@ -149,34 +155,41 @@ matrix: - libc6-dev-i386 - gcc-multilib - gcc-4.4 + script: + - make clean all CC=gcc-4.4 MOREFLAGS=-Werror + - make clean + - CFLAGS=-fPIC LDFLAGS='-pie -fPIE -D_FORTIFY_SOURCE=2' make -C programs # tag-specific test - - if: tag =~ ^v[0-9]\.[0-9] + - name: tag build + if: tag =~ ^v[0-9]\.[0-9] os: linux - sudo: false - env: Cmd="make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH " COMPILER=cc - - - dist: xenial - sudo: required - env: BUILD_SYSTEM='meson' + script: + - make -C tests checkTag + - tests/checkTag "$TRAVIS_BRANCH" + + - name: (Xenial) Meson + clang build + env: ALLOW_FAILURES=true + dist: xenial + language: cpp + compiler: clang + install: + - sudo apt-get install -qq python3 tree + - curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip' + && unzip ~/ninja.zip -d ~/.local/bin + - curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' + && python3 ~/get-pip.py --user + && pip3 install --user meson + script: + - meson setup + --buildtype=debug + -Db_lundef=false + -Dauto_features=enabled + -Ddefault_library=both + -Dbuild_{programs,contrib,tests,examples}=true + contrib/meson build + - cd build + - DESTDIR=./staging ninja install + - tree ./staging allow_failures: - - env: BUILD_SYSTEM='meson' - -script: - - if [ "${BUILD_SYSTEM}" = meson ]; then - sudo apt-get install -qq python3 tree - && curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" - && python3 get-pip.py --user && rm get-pip.py - && pip3 install --user meson ninja - && meson --buildtype=debug -Dauto_features=enabled -Ddefault_library=both - -Dbuild_{programs,contrib,tests,examples}=true contrib/meson build - && cd "$_" - && ninja - && DESTDIR=./staging ninja install - && tree ./staging; - travis_terminate "$?"; - fi - - uname -a - - echo Cmd=$Cmd - - $COMPILER -v - - sh -c "$Cmd" + - env: ALLOW_FAILURES=true diff --git a/contrib/meson/GetLz4LibraryVersion.py b/contrib/meson/GetLz4LibraryVersion.py index e929f95..d8abfcb 100644 --- a/contrib/meson/GetLz4LibraryVersion.py +++ b/contrib/meson/GetLz4LibraryVersion.py @@ -8,15 +8,9 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# import re -import sys -def usage(): - print('usage: python3 GetLz4LibraryVersion.py <path/to/lz4.h>') - sys.exit(1) - - -def find_version(filepath): +def find_version_tuple(filepath): version_file_data = None with open(filepath) as fd: version_file_data = fd.read() @@ -33,12 +27,12 @@ def find_version(filepath): def main(): - if len(sys.argv) < 2: - usage() - - filepath = sys.argv[1] - version_tup = find_version(filepath) - print('.'.join(version_tup)) + import argparse + parser = argparse.ArgumentParser(description='Print lz4 version from lib/lz4.h') + parser.add_argument('file', help='path to lib/lz4.h') + args = parser.parse_args() + version_tuple = find_version_tuple(args.file) + print('.'.join(version_tuple)) if __name__ == '__main__': diff --git a/contrib/meson/InstallSymlink.py b/contrib/meson/InstallSymlink.py index d7b1e5a..3f2998c 100644 --- a/contrib/meson/InstallSymlink.py +++ b/contrib/meson/InstallSymlink.py @@ -1,71 +1,54 @@ #!/usr/bin/env python3 # ############################################################################# -# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com> +# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com> # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the # LICENSE file in the root directory of this source tree) and the GPLv2 (found # in the COPYING file in the root directory of this source tree). # ############################################################################# -import errno -import os - +# This file should be synced with https://github.com/lzutao/meson-symlink -def mkdir_p(path, dir_mode=0o777): - try: - os.makedirs(path, mode=dir_mode) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise +import os +import pathlib # since Python 3.4 -def InstallSymlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): - if not os.path.exists(install_dir): - mkdir_p(install_dir, dir_mode) - if not os.path.isdir(install_dir): - raise NotADirectoryError(install_dir) +def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): + if not install_dir.exists(): + install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True) + if not install_dir.is_dir(): + raise NotADirectoryError(install_dir) - new_dst = os.path.join(install_dir, dst) - if os.path.islink(new_dst) and os.readlink(new_dst) == src: - print('File exists: %r -> %r' % (dst, src)) + new_dst = install_dir.joinpath(dst) + if new_dst.is_symlink() and os.readlink(new_dst) == src: + print('File exists: {!r} -> {!r}'.format(new_dst, src)) return - print('Installing symlink %r -> %r' % (new_dst, src)) - os.symlink(src, new_dst, dst_is_dir) + print('Installing symlink {!r} -> {!r}'.format(new_dst, src)) + new_dst.symlink_to(src, target_is_directory=dst_is_dir) def main(): import argparse - parser = argparse.ArgumentParser(description='Install a symlink.\n', - usage='usage: InstallSymlink.py [-h] [-d] [-m MODE] src dst ' - 'install_dir\n\n' + parser = argparse.ArgumentParser(description='Install a symlink', + usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n' 'example:\n' - '\tInstallSymlink.py libcrypto.so.1.0.0 libcrypt.so ' - '/usr/lib/x86_64-linux-gnu False') - parser.add_argument('src', help='target to link') - parser.add_argument('dst', help='link name') + ' {0} dash sh /bin'.format(pathlib.Path(__file__).name)) + parser.add_argument('source', help='target to link') + parser.add_argument('dest', help='link name') parser.add_argument('install_dir', help='installation directory') parser.add_argument('-d', '--isdir', action='store_true', - help='dst is a directory') + help='dest is a directory') parser.add_argument('-m', '--mode', help='directory mode on creating if not exist', - default='0o777') + default='0o755') args = parser.parse_args() - src = args.src - dst = args.dst - install_dir = args.install_dir - dst_is_dir = args.isdir dir_mode = int(args.mode, 8) - DESTDIR = os.environ.get('DESTDIR') - if DESTDIR: - install_dir = DESTDIR + install_dir if os.path.isabs(install_dir) \ - else os.path.join(DESTDIR, install_dir) - - InstallSymlink(src, dst, install_dir, dst_is_dir, dir_mode) + meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='') + install_dir = pathlib.Path(meson_destdir, args.install_dir) + install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode) if __name__ == '__main__': diff --git a/contrib/meson/README.md b/contrib/meson/README.md new file mode 100644 index 0000000..fa18493 --- /dev/null +++ b/contrib/meson/README.md @@ -0,0 +1,34 @@ +Meson build system for lz4 +========================== + +Meson is a build system designed to optimize programmer productivity. +It aims to do this by providing simple, out-of-the-box support for +modern software development tools and practices, such as unit tests, +coverage reports, Valgrind, CCache and the like. + +This Meson build system is provided with no guarantee. + +## How to build + +`cd` to this meson directory (`contrib/meson`) + +```sh +meson setup --buildtype=release -Ddefault_library=shared -Dbuild_programs=true builddir +cd builddir +ninja # to build +ninja install # to install +``` + +You might want to install it in staging directory: + +```sh +DESTDIR=./staging ninja install +``` + +To configure build options, use: + +```sh +meson configure +``` + +See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). diff --git a/contrib/meson/lib/meson.build b/contrib/meson/lib/meson.build index 3810601..e782334 100644 --- a/contrib/meson/lib/meson.build +++ b/contrib/meson/lib/meson.build @@ -27,19 +27,23 @@ if use_debug endif liblz4_c_args += cc.get_supported_arguments(liblz4_debug_cflags) +if host_machine_os == os_windows and default_library != 'static' + liblz4_c_args += '-DLZ4_DLL_EXPORT=1' +endif + liblz4 = library('lz4', liblz4_sources, include_directories: liblz4_includes, c_args: liblz4_c_args, install: true, - soversion: lz4_libversion) + version: lz4_libversion) liblz4_dep = declare_dependency(link_with: liblz4, include_directories: liblz4_includes) -pkgconfig.generate(name: 'lz4', - filebase: 'lz4', - libraries: [liblz4], +pkgconfig.generate(liblz4, + name: 'lz4', + filebase: 'liblz4', description: 'extremely fast lossless compression algorithm library', version: lz4_libversion, url: 'http://www.lz4.org/') @@ -47,6 +51,7 @@ pkgconfig.generate(name: 'lz4', install_headers(join_paths(lz4_root_dir, 'lib/lz4.h'), join_paths(lz4_root_dir, 'lib/lz4hc.h'), join_paths(lz4_root_dir, 'lib/lz4frame.h')) -if get_option('default_library') != 'shared' + +if default_library != 'shared' install_headers(join_paths(lz4_root_dir, 'lib/lz4frame_static.h')) endif diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index f1ca503..bf30eae 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -17,6 +17,8 @@ project('lz4', ['c'], cc = meson.get_compiler('c') pkgconfig = import('pkgconfig') python3 = import('python').find_installation() +c_std = get_option('c_std') +default_library = get_option('default_library') host_machine_os = host_machine.system() os_windows = 'windows' @@ -31,26 +33,30 @@ compiler_clang = 'clang' compiler_msvc = 'msvc' lz4_version = meson.project_version() -lz4_libversion = '' -c_std = get_option('c_std') +lz4_h_file = join_paths(meson.current_source_dir(), '../../lib/lz4.h') +GetLz4LibraryVersion_py = files('GetLz4LibraryVersion.py') +r = run_command(python3, GetLz4LibraryVersion_py, lz4_h_file) +if r.returncode() == 0 + output = r.stdout().strip() + if output.version_compare('>@0@'.format(lz4_version)) + lz4_version = output + message('Project version is now: @0@'.format(lz4_version)) + endif +else + warning('Cannot find project version in @0@'.format(lz4_h_file)) +endif + +lz4_libversion = lz4_version # ============================================================================= # Installation directories # ============================================================================= -if host_machine_os == os_windows - lz4_prefix = '.' - lz4_bindir = 'bin' - lz4_datadir = 'share' - lz4_mandir = join_paths(lz4_datadir, 'man') -else - lz4_prefix = get_option('prefix') - lz4_bindir = join_paths(lz4_prefix, get_option('bindir')) - lz4_datadir = join_paths(lz4_prefix, get_option('datadir')) - lz4_mandir = join_paths(lz4_prefix, get_option('mandir')) -endif - +lz4_prefix = get_option('prefix') +lz4_bindir = get_option('bindir') +lz4_datadir = get_option('datadir') +lz4_mandir = get_option('mandir') lz4_docdir = join_paths(lz4_datadir, 'doc', meson.project_name()) # ============================================================================= @@ -73,30 +79,6 @@ build_examples = get_option('build_examples') #feature_multi_thread = get_option('multi_thread') # ============================================================================= -# Helper scripts for Meson -# ============================================================================= - -GetLz4LibraryVersion_py = files('GetLz4LibraryVersion.py') - -# ============================================================================= -# Getting project version from lz4.h -# ============================================================================= - -lz4_h_file = join_paths(meson.current_source_dir(), 'lib/lz4.h') -r = run_command(python3, GetLz4LibraryVersion_py, lz4_h_file) -if r.returncode() == 0 - output = r.stdout().strip() - if output.version_compare('>@0@'.format(lz4_version)) - lz4_version = output - message('Project version is now: @0@'.format(lz4_version)) - endif -endif - -if host_machine_os != os_windows - lz4_libversion = lz4_version -endif - -# ============================================================================= # Dependencies # ============================================================================= @@ -112,9 +94,8 @@ add_project_arguments(['-DXXH_NAMESPACE=LZ4_'], language: 'c') if [compiler_gcc, compiler_clang].contains(cc_id) common_warning_flags = [] - if buildtype == 'release' - common_warning_flags = ['-Werror'] - endif + # Should use Meson's own --werror build option + #common_warning_flags += ['-Werror'] if c_std == 'c89' or c_std == 'gnu89' common_warning_flags += ['-pedantic', '-Wno-long-long', '-Wno-variadic-macros'] elif c_std == 'c99' or c_std == 'gnu99' diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index b5c3228..df64eb0 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -43,9 +43,10 @@ install_man(join_paths(lz4_root_dir, 'programs/lz4.1')) InstallSymlink_py = '../InstallSymlink.py' lz4_man1_dir = join_paths(lz4_mandir, 'man1') -man1_EXT = host_machine_os != os_windows ? '.1.gz' : '.1' +bin_EXT = host_machine_os == os_windows ? '.exe' : '' +man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz' foreach f : ['lz4c', 'lz4cat', 'unlz4'] - meson.add_install_script(InstallSymlink_py, 'lz4', f, lz4_bindir) + meson.add_install_script(InstallSymlink_py, 'lz4' + bin_EXT, f + bin_EXT, lz4_bindir) meson.add_install_script(InstallSymlink_py, 'lz4' + man1_EXT, f + man1_EXT, lz4_man1_dir) endforeach diff --git a/contrib/meson/tests/meson.build b/contrib/meson/tests/meson.build index 82e2813..392bcf2 100644 --- a/contrib/meson/tests/meson.build +++ b/contrib/meson/tests/meson.build @@ -15,7 +15,7 @@ lib_dir_inc = include_directories(join_paths(lz4_root_dir, 'lib')) # Test flags # ============================================================================= -TEST_FILES = join_paths(lz4_root_dir, 'tests/COPYING') +TEST_FILES = join_paths(meson.current_source_dir(), lz4_root_dir, 'tests/COPYING') FUZZER_TIME = '-T90s' NB_LOOPS = '-i1' @@ -76,6 +76,18 @@ checkTag = executable('checkTag', # Tests (Use "meson test --list" to list all tests) # ============================================================================= -test('test-fullbench', fullbench, args: ['--no-prompt', NB_LOOPS, TEST_FILES]) -test('test-fuzzer', fuzzer, args: [FUZZER_TIME]) -test('test-frametest', frametest, args: [FUZZER_TIME]) +# XXX: (Need TEST) These timeouts (in seconds) when running on a HDD should be +# at least six times bigger than on a SSD + +test('test-fullbench', + fullbench, + args: ['--no-prompt', NB_LOOPS, TEST_FILES], + timeout: 420) # Should enough when running on HDD +test('test-fuzzer', + fuzzer, + args: [FUZZER_TIME], + timeout: 100) +test('test-frametest', + frametest, + args: [FUZZER_TIME], + timeout: 100) diff --git a/contrib/snap/README.md b/contrib/snap/README.md new file mode 100644 index 0000000..612d6d7 --- /dev/null +++ b/contrib/snap/README.md @@ -0,0 +1,29 @@ +Snap Packaging +-------------- + +This directory contains the config required to generate a snap package +of lz4. Snaps are universal Linux packages that allow you to easily +build your application from any source and ship it to any Linux +distribution by publishing it to https://snapcraft.io/. A key attribute +of a snap package is that it is (ideally) confined such that it +executes within a controlled environmenti with all its dependencies +bundled with it and does not share dependencies with of from any other +package on the system (with a couple of minor exceptions). + +The basic anatomy and workflow is: + + * ensure snap.snapcraft.yaml is up-to-date e.g. with version info + + * build the snap by installing the snapcraft package and running it + + * push snap/* changes to the repo (excluding any crud generated by a build of course) + + * register yourself as owner of lz4 name in snapstore + + * publish new snap to the snap store + + * install snap by doing 'snap install lz4' on any Linux distro + + * all installed copies of lz4 will be automatically updated to your new version + +For more information on Snaps see https://docs.snapcraft.io and https://forum.snapcraft.io/ diff --git a/contrib/snap/snapcraft.yaml b/contrib/snap/snapcraft.yaml new file mode 100644 index 0000000..2793c0e --- /dev/null +++ b/contrib/snap/snapcraft.yaml @@ -0,0 +1,31 @@ +name: lz4 +version: 1.8.4 +summary: Extremely Fast Compression algorithm +description: > + LZ4 is lossless compression algorithm, providing compression + speed > 500 MB/s per core, scalable with multi-cores CPU. It features an + extremely fast decoder, with speed in multiple GB/s per core, typically + reaching RAM speed limits on multi-core systems. + . + Speed can be tuned dynamically, selecting an "acceleration" factor which + trades compression ratio for faster speed. On the other end, a high + compression derivative, LZ4_HC, is also provided, trading CPU time for + improved compression ratio. All versions feature the same decompression + speed. + . + LZ4 is also compatible with dictionary compression, and can ingest any + input file as dictionary, including those created by Zstandard Dictionary + Builder. (note: only the final 64KB are used). + . + LZ4 library is provided as open-source software using BSD 2-Clause license. +confinement: strict +grade: stable + +apps: + lz4: + command: usr/local/bin/lz4 + plugs: [home] +parts: + lz4: + source: ../ + plugin: make diff --git a/doc/lz4_Block_format.md b/doc/lz4_Block_format.md index 5438730..2fb4c19 100644 --- a/doc/lz4_Block_format.md +++ b/doc/lz4_Block_format.md @@ -1,6 +1,6 @@ LZ4 Block Format Description ============================ -Last revised: 2018-04-25. +Last revised: 2018-12-30. Author : Yann Collet @@ -10,7 +10,8 @@ using any programming language. LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. There is no entropy encoder back-end nor framing layer. -The latter is assumed to be handled by other parts of the system (see [LZ4 Frame format]). +The latter is assumed to be handled by other parts of the system +(see [LZ4 Frame format]). This design is assumed to favor simplicity and speed. It helps later on for optimizations, compactness, and features. @@ -104,45 +105,41 @@ A common case is an offset of 1, meaning the last byte is repeated `matchlength` times. -Parsing restrictions +End of block restrictions ----------------------- -There are specific parsing rules to respect in order to remain compatible -with assumptions made by the decoder : - -1. The last 5 bytes are always literals. In other words, the last five bytes - from the uncompressed input (or all bytes, if the input has less than five - bytes) must be encoded as literals on behalf of the last sequence. - The last sequence is incomplete, and stops right after the literals. -2. The last match must start at least 12 bytes before end of block. - The last match is part of the penultimate sequence, - since the last sequence stops right after literals. +There are specific rules required to terminate a block. + +1. The last sequence only contains literals. The block ends right after them. +1. The last 5 bytes of input are always literals. + Therefore, the last sequence contains at least 5 bytes, + or all input bytes if input is smaller than 5 bytes + (empty input can be represented with a zero byte, + interpreted as a token without literal and without a match). +2. The last match must start at least 12 bytes before the end of block. + The last match is part of the penultimate sequence. + It is followed by the last sequence, which only contains literals. Note that, as a consequence, blocks < 13 bytes cannot be compressed. -These rules are in place to ensure that the decoder -can speculatively execute copy instructions -without ever reading nor writing beyond provided I/O buffers. - -1. To copy literals from a non-last sequence, an 8-byte copy instruction - can always be safely issued (without reading past the input), - because literals are followed by a 2-byte offset, - and last sequence is at least 1+5 bytes long. -2. Similarly, a match operation can speculatively copy up to 12 bytes - while remaining within output buffer boundaries. - -Empty inputs can be represented with a zero byte, -interpreted as a token without literals and without a match. +These rules are in place to ensure that a compatible decoder +can be designed for speed, issuing speculatively instructions, +while never reading nor writing beyond provided I/O buffers. Additional notes ----------------------- -There is no assumption nor limits to the way the compressor +If the decoder will decompress data from an external source, +it is recommended to ensure that the decoder will not be vulnerable to +buffer overflow manipulations. +Always ensure that read and write operations +remain within the limits of provided buffers. +Test the decoder with fuzzers +to ensure it's resilient to improbable combinations. + +The format makes no assumption nor limits to the way the compressor searches and selects matches within the source data block. -It could be a fast scan, a multi-probe, a full search using BST, -standard hash chains or MMC, well whatever. - -Advanced parsing strategies can also be implemented, such as lazy match, -or full optimal parsing. - -All these trade-off offer distinctive speed/memory/compression advantages. -Whatever the method used by the compressor, its result will be decodable -by any LZ4 decoder if it follows the format specification described above. +Multiple techniques can be considered, +featuring distinct time / performance trade offs. +As long as the format is respected, +the result will be compatible and decodable by any compliant decoder. +An upper compression limit can be reached, +using a technique called "full optimal parsing", at high cpu cost. diff --git a/doc/lz4_Frame_format.md b/doc/lz4_Frame_format.md index a8541f5..a0514e0 100644 --- a/doc/lz4_Frame_format.md +++ b/doc/lz4_Frame_format.md @@ -265,20 +265,23 @@ The highest bit is “1” if data in the block is uncompressed. The highest bit is “0” if data in the block is compressed by LZ4. -All other bits give the size, in bytes, of the following data block -(the size does not include the block checksum if present). +All other bits give the size, in bytes, of the following data block. +The size does not include the block checksum if present. Block Size shall never be larger than Block Maximum Size. -Such a thing could happen for incompressible source data. -In such case, such a data block shall be passed in uncompressed format. +Such a thing could potentially happen for non-compressible sources. +In such a case, such data block shall be passed using uncompressed format. __Data__ Where the actual data to decode stands. It might be compressed or not, depending on previous field indications. -Uncompressed size of Data can be any size, up to “block maximum size”. -Note that data block is not necessarily full : -an arbitrary “flush” may happen anytime. Any block can be “partially filled”. + +When compressed, the data must respect the [LZ4 block format specification](https://github.com/lz4/lz4/blob/master/doc/lz4_Block_format.md). + +Note that the block is not necessarily full. +Uncompressed size of data can be any size, up to "Block Maximum Size”, +so it may contain less data than the maximum block size. __Block checksum__ @@ -143,7 +143,7 @@ * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute * of LZ4_wildCopy does not affect the compression speed. */ -#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) # define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE #else @@ -297,7 +297,76 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) do { memcpy(d,s,8); d+=8; s+=8; } while (d<e); } +static const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; +static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; + +#if defined(__i386__) || defined(__x86_64__) +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { + if (offset < 8) { + dstPtr[0] = srcPtr[0]; + dstPtr[1] = srcPtr[1]; + dstPtr[2] = srcPtr[2]; + dstPtr[3] = srcPtr[3]; + srcPtr += inc32table[offset]; + memcpy(dstPtr+4, srcPtr, 4); + srcPtr -= dec64table[offset]; + dstPtr += 8; + } else { + memcpy(dstPtr, srcPtr, 8); + dstPtr += 8; + srcPtr += 8; + } + + LZ4_wildCopy(dstPtr, srcPtr, dstEnd); +} + +/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd */ +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { memcpy(d,s,16); memcpy(d+16,s+16,16); d+=32; s+=32; } while (d<e); +} +LZ4_FORCE_O2_INLINE_GCC_PPC64LE +void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { + BYTE v[8]; + switch(offset) { + case 1: + memset(v, *srcPtr, 8); + goto copy_loop; + case 2: + memcpy(v, srcPtr, 2); + memcpy(&v[2], srcPtr, 2); + memcpy(&v[4], &v[0], 4); + goto copy_loop; + case 4: + memcpy(v, srcPtr, 4); + memcpy(&v[4], srcPtr, 4); + goto copy_loop; + case 3: + case 5: + case 6: + case 7: + case 8: + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + copy_loop: + memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif /*-************************************ * Common Constants **************************************/ @@ -307,6 +376,7 @@ void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) #define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ #define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ #define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 static const int LZ4_minLength = (MFLIMIT+1); #define KB *(1 <<10) @@ -1319,7 +1389,7 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* invalidate tiny dictionaries */ - if ( (streamPtr->dictSize-1 < 4) /* intentional underflow */ + if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ && (dictEnd != (const BYTE*)source) ) { DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); streamPtr->dictSize = 0; @@ -1434,6 +1504,35 @@ typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; #undef MIN #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { loop_error = -2, initial_error = -1, ok = 0} variable_length_error; +LZ4_FORCE_INLINE unsigned read_variable_length(const BYTE**ip, const BYTE* lencheck, int loop_check, int initial_check, variable_length_error* error) { + unsigned length = 0; + unsigned s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); + + return length; +} + /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. @@ -1465,16 +1564,21 @@ LZ4_decompress_generic( BYTE* cpy; const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; - const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; - const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + /* Set up the "end" pointers for the shortcut. */ const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); /* Special cases */ @@ -1483,13 +1587,135 @@ LZ4_decompress_generic( if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0 ? 1 : -1); if ((endOnInput) && unlikely(srcSize==0)) return -1; - /* Main Loop : decode sequences */ + /* Currently the fast loop shows a regression on qualcomm arm chips. */ +#if defined(__i386__) || defined(__x86_64__) + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) + goto safe_decode; + + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ while (1) { - const BYTE* match; - size_t offset; + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-FASTLOOP_SAFE_DISTANCE) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-FASTLOOP_SAFE_DISTANCE)) ) + { + goto safe_literal_copy; + } + LZ4_wildCopy32(op, ip, cpy); + ip += length; op = cpy; + } else { + cpy = op+length; + /* We don't need to check oend, since we check it once for each loop below */ + if ( ((endOnInput) && (ip+16>iend-(2+1+LASTLITERALS)))) + { + goto safe_literal_copy; + } + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + memcpy(op, ip, 16); + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; - unsigned const token = *ip++; - size_t length = token >> ML_BITS; /* literal length */ + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if (!(dict == usingExtDict) || (match >= lowPrefix)) { + if (offset >= 8) { + memcpy(op, match, 8); + memcpy(op+8, match+8, 8); + memcpy(op+16, match+16, 2); + op += length; + continue; + } + } + } + + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may not respect endBlock parsing restrictions */ + assert(op<=oend); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ @@ -1536,18 +1762,18 @@ LZ4_decompress_generic( /* decode literal length */ if (length == RUN_MASK) { - unsigned s; - if (unlikely(endOnInput ? ip >= iend-RUN_MASK : 0)) goto _output_error; /* overflow detection */ - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip<iend-RUN_MASK : 1) & (s==255) ); + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, endOnInput, endOnInput, &error); + if (error == initial_error) goto _output_error; if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) goto _output_error; /* overflow detection */ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) goto _output_error; /* overflow detection */ } /* copy literals */ cpy = op+length; +#if defined(__i386__) || defined(__x86_64__) + safe_literal_copy: +#endif LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) @@ -1588,16 +1814,16 @@ LZ4_decompress_generic( } /* note : when partialDecoding, there is no guarantee that at least 4 bytes remain available in output buffer */ if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, endOnInput, 0, &error); + if (error != ok) goto _output_error; if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ } length += MINMATCH; +#if defined(__i386__) || defined(__x86_64__) + safe_match_copy: +#endif /* match starting within external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) { diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 705832d..3f81ef0 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -265,22 +265,21 @@ unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } - -/*-************************************ -* Private functions -**************************************/ -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) - -static size_t LZ4F_getBlockSize(unsigned blockSizeID) +size_t LZ4F_getBlockSize(unsigned blockSizeID) { static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (blockSizeID < 4 || blockSizeID > 7) return err0r(LZ4F_ERROR_maxBlockSize_invalid); blockSizeID -= 4; - if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid); return blockSizes[blockSizeID]; } +/*-************************************ +* Private functions +**************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + static BYTE LZ4F_headerChecksum (const void* header, size_t length) { U32 const xxh = XXH32(header, length, 0); diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 7c7c34e..68f4118 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -483,6 +483,7 @@ typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned); /********************************** * Bulk processing dictionary API diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 56c8f47..129bf0c 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -88,6 +88,8 @@ typedef enum { #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } @@ -437,7 +439,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode Literal length */ length = (size_t)(*ip - *anchor); - if ((limit) && ((*op + (length >> 8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length >= RUN_MASK) { size_t len = length - RUN_MASK; *token = (RUN_MASK << ML_BITS); @@ -458,7 +460,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( /* Encode MatchLength */ assert(matchLength >= MINMATCH); length = (size_t)(matchLength - MINMATCH); - if ((limit) && (*op + (length >> 8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length >= ML_MASK) { *token += ML_MASK; length -= ML_MASK; @@ -533,7 +535,7 @@ _Search2: if (ml2 == ml) { /* No better match => encode ML1 */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; continue; } @@ -581,10 +583,10 @@ _Search3: if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start2; optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow; continue; } @@ -603,7 +605,7 @@ _Search3: } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; ip = start3; ref = ref3; ml = ml3; @@ -641,7 +643,7 @@ _Search3: } } optr = op; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) goto _dest_overflow; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow; /* ML2 becomes ML1 */ ip = start2; ref = ref2; ml = ml2; @@ -1206,7 +1208,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, int const firstML = firstMatch.len; const BYTE* const matchPos = ip - firstMatch.off; opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; continue; } @@ -1378,7 +1380,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, assert(ml >= MINMATCH); assert((offset >= 1) && (offset <= MAX_DISTANCE)); opSaved = op; - if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */ goto _dest_overflow; } } } /* while (ip <= mflimit) */ diff --git a/programs/lz4cli.c b/programs/lz4cli.c index 3709f50..464e43b 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -92,7 +92,7 @@ static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : dow ***************************************/ #define DEFAULT_COMPRESSOR LZ4IO_compressFilename #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ +int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ /*-*************************** @@ -287,6 +287,19 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; +/** determineOpMode() : + * auto-determine operation mode, based on input filename extension + * @return `om_decompress` if input filename has .lz4 extension and `om_compress` otherwise. + */ +static operationMode_e determineOpMode(const char* inputFilename) +{ + size_t const inSize = strlen(inputFilename); + size_t const extSize = strlen(LZ4_EXTENSION); + size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; + if (!strcmp(inputFilename+extStart, LZ4_EXTENSION)) return om_decompress; + else return om_compress; +} + int main(int argc, const char** argv) { int i, @@ -305,9 +318,10 @@ int main(int argc, const char** argv) char* dynNameSpace = NULL; const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); unsigned ifnIdx=0; + LZ4IO_prefs_t* const prefs = LZ4IO_defaultPreferences(); const char nullOutput[] = NULL_OUTPUT; const char extension[] = LZ4_EXTENSION; - size_t blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT); + size_t blockSize = LZ4IO_setBlockSizeID(prefs, LZ4_BLOCKSIZEID_DEFAULT); const char* const exeName = lastNameFromPath(argv[0]); #ifdef UTIL_HAS_CREATEFILELIST const char** extendedFileList = NULL; @@ -321,13 +335,14 @@ int main(int argc, const char** argv) return 1; } inFileNames[0] = stdinmark; - LZ4IO_setOverwrite(0); + LZ4IO_setOverwrite(prefs, 0); /* predefined behaviors, based on binary/link name */ if (exeNameMatch(exeName, LZ4CAT)) { mode = om_decompress; - LZ4IO_setOverwrite(1); - LZ4IO_setRemoveSrcFile(0); + LZ4IO_setOverwrite(prefs, 1); + LZ4IO_setPassThrough(prefs, 1); + LZ4IO_setRemoveSrcFile(prefs, 0); forceStdout=1; output_filename=stdoutmark; displayLevel=1; @@ -359,23 +374,23 @@ int main(int argc, const char** argv) || (!strcmp(argument, "--uncompress"))) { mode = om_decompress; continue; } if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; continue; } if (!strcmp(argument, "--test")) { mode = om_test; continue; } - if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; } - if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; } + if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(prefs, 1); continue; } + if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(prefs, 0); continue; } if ((!strcmp(argument, "--stdout")) || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; continue; } - if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; } - if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; } - if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; } - if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(0); continue; } - if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(2); continue; } - if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(0); continue; } - if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(1); continue; } + if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 1); continue; } + if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); continue; } + if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; } + if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; } + if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(prefs, 2); continue; } + if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(prefs, 0); continue; } + if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; } if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--help")) { usage_advanced(exeName); goto _cleanup; } - if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(0); continue; } /* keep source file (default) */ - if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(1); continue; } + if (!strcmp(argument, "--keep")) { LZ4IO_setRemoveSrcFile(prefs, 0); continue; } /* keep source file (default) */ + if (!strcmp(argument, "--rm")) { LZ4IO_setRemoveSrcFile(prefs, 1); continue; } if (longCommandWArg(&argument, "--fast")) { /* Parse optional acceleration factor */ if (*argument == '=') { @@ -406,7 +421,7 @@ int main(int argc, const char** argv) if (!strcmp(argument, "c1")) { cLevel=9; argument++; continue; } /* -c1 (high compression) */ if (!strcmp(argument, "c2")) { cLevel=12; argument++; continue; } /* -c2 (very high compression) */ if (!strcmp(argument, "hc")) { cLevel=12; argument++; continue; } /* -hc (very high compression) */ - if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(1); continue; } /* -y (answer 'yes' to overwrite permission) */ + if (!strcmp(argument, "y")) { LZ4IO_setOverwrite(prefs, 1); continue; } /* -y (answer 'yes' to overwrite permission) */ } if ((*argument>='0') && (*argument<='9')) { @@ -455,13 +470,17 @@ int main(int argc, const char** argv) case 'd': mode = om_decompress; break; /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; output_filename=stdoutmark; break; + case 'c': + forceStdout=1; + output_filename=stdoutmark; + LZ4IO_setPassThrough(prefs, 1); + break; /* Test integrity */ case 't': mode = om_test; break; /* Overwrite */ - case 'f': LZ4IO_setOverwrite(1); break; + case 'f': LZ4IO_setOverwrite(prefs, 1); break; /* Verbose mode */ case 'v': displayLevel++; break; @@ -470,7 +489,7 @@ int main(int argc, const char** argv) case 'q': if (displayLevel) displayLevel--; break; /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ - case 'k': LZ4IO_setRemoveSrcFile(0); break; + case 'k': LZ4IO_setRemoveSrcFile(prefs, 0); break; /* Modify Block Properties */ case 'B': @@ -478,8 +497,8 @@ int main(int argc, const char** argv) int exitBlockProperties=0; switch(argument[1]) { - case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break; - case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */ + case 'D': LZ4IO_setBlockMode(prefs, LZ4IO_blockLinked); argument++; break; + case 'X': LZ4IO_setBlockChecksumMode(prefs, 1); argument ++; break; /* disabled by default */ default : if (argument[1] < '0' || argument[1] > '9') { exitBlockProperties=1; @@ -491,12 +510,12 @@ int main(int argc, const char** argv) argument--; if (B < 4) badusage(exeName); if (B <= 7) { - blockSize = LZ4IO_setBlockSizeID(B); + blockSize = LZ4IO_setBlockSizeID(prefs, B); BMK_setBlockSize(blockSize); DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); } else { if (B < 32) badusage(exeName); - blockSize = LZ4IO_setBlockSize(B); + blockSize = LZ4IO_setBlockSize(prefs, B); BMK_setBlockSize(blockSize); if (blockSize >= 1024) { DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); @@ -605,7 +624,7 @@ int main(int argc, const char** argv) } if (mode == om_test) { - LZ4IO_setTestMode(1); + LZ4IO_setTestMode(prefs, 1); output_filename = nulmark; mode = om_decompress; /* defer to decompress */ } @@ -615,7 +634,7 @@ int main(int argc, const char** argv) DISPLAYLEVEL(1, "refusing to read from a console\n"); exit(1); } - LZ4IO_setDictionaryFilename(dictionary_filename); + LZ4IO_setDictionaryFilename(prefs, dictionary_filename); } /* compress or decompress */ @@ -633,11 +652,7 @@ int main(int argc, const char** argv) while ((!output_filename) && (multiple_inputs==0)) { if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ - size_t const inSize = strlen(input_filename); - size_t const extSize = strlen(LZ4_EXTENSION); - size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; - if (!strcmp(input_filename+extStart, LZ4_EXTENSION)) mode = om_decompress; - else mode = om_compress; + mode = determineOpMode(input_filename); } if (mode == om_compress) { /* compression to file */ size_t const l = strlen(input_filename); @@ -675,23 +690,28 @@ int main(int argc, const char** argv) if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; + /* Auto-determine compression or decompression, based on file extension */ + if (mode == om_auto) { + mode = determineOpMode(input_filename); + } + /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); if (ifnIdx == 0) multiple_inputs = 0; if (mode == om_decompress) { if (multiple_inputs) - operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); + operationResult = LZ4IO_decompressMultipleFilenames(prefs, inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LZ4_EXTENSION); else - operationResult = DEFAULT_DECOMPRESSOR(input_filename, output_filename); + operationResult = DEFAULT_DECOMPRESSOR(prefs, input_filename, output_filename); } else { /* compression is default action */ if (legacy_format) { DISPLAYLEVEL(3, "! Generating LZ4 Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); + LZ4IO_compressFilename_Legacy(prefs, input_filename, output_filename, cLevel); } else { if (multiple_inputs) - operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); + operationResult = LZ4IO_compressMultipleFilenames(prefs, inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); else - operationResult = DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + operationResult = DEFAULT_COMPRESSOR(prefs, input_filename, output_filename, cLevel); } } @@ -704,6 +724,7 @@ _cleanup: inFileNames = NULL; } #endif + LZ4IO_freePreferences(prefs); free((void*)inFileNames); return operationResult; } diff --git a/programs/lz4io.c b/programs/lz4io.c index a35928d..6bb6c48 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -107,19 +107,23 @@ static clock_t g_time = 0; /************************************** * Local Parameters **************************************/ -static int g_overwrite = 1; -static int g_testMode = 0; -static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; -static size_t g_blockSize = 0; -static int g_blockChecksum = 0; -static int g_streamChecksum = 1; -static int g_blockIndependence = 1; -static int g_sparseFileSupport = 1; -static int g_contentSizeFlag = 0; -static int g_useDictionary = 0; -static unsigned g_favorDecSpeed = 0; -static const char* g_dictionaryFilename = NULL; +struct LZ4IO_prefs_s { + int passThrough; + int overwrite; + int testMode; + int blockSizeId; + size_t blockSize; + int blockChecksum; + int streamChecksum; + int blockIndependence; + int sparseFileSupport; + int contentSizeFlag; + int useDictionary; + unsigned favorDecSpeed; + char const* dictionaryFilename; + U32 removeSrcFile; +}; /************************************** * Exceptions @@ -151,73 +155,109 @@ static const char* g_dictionaryFilename = NULL; /* ****************** Parameters ******************** */ /* ************************************************** */ -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename) { - g_dictionaryFilename = dictionaryFilename; - g_useDictionary = dictionaryFilename != NULL; - return g_useDictionary; +LZ4IO_prefs_t* LZ4IO_defaultPreferences(void) +{ + LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(LZ4IO_prefs_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + ret->passThrough = 0; + ret->overwrite = 1; + ret->testMode = 0; + ret->blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; + ret->blockSize = 0; + ret->blockChecksum = 0; + ret->streamChecksum = 1; + ret->blockIndependence = 1; + ret->sparseFileSupport = 1; + ret->contentSizeFlag = 0; + ret->useDictionary = 0; + ret->favorDecSpeed = 0; + ret->dictionaryFilename = NULL; + ret->removeSrcFile = 0; + return ret; +} + +void LZ4IO_freePreferences(LZ4IO_prefs_t* const prefs) +{ + free(prefs); +} + + +int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename) +{ + prefs->dictionaryFilename = dictionaryFilename; + prefs->useDictionary = dictionaryFilename != NULL; + return prefs->useDictionary; +} + +/* Default setting : passThrough = 0; return : passThrough mode (0/1) */ +int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes) +{ + prefs->passThrough = (yes!=0); + return prefs->passThrough; } + /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes) +int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes) { - g_overwrite = (yes!=0); - return g_overwrite; + prefs->overwrite = (yes!=0); + return prefs->overwrite; } /* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes) +int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes) { - g_testMode = (yes!=0); - return g_testMode; + prefs->testMode = (yes!=0); + return prefs->testMode; } /* blockSizeID : valid values : 4-5-6-7 */ -size_t LZ4IO_setBlockSizeID(unsigned bsid) +size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid) { static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB }; static const unsigned minBlockSizeID = 4; static const unsigned maxBlockSizeID = 7; if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - g_blockSizeId = bsid; - g_blockSize = blockSizeTable[g_blockSizeId-minBlockSizeID]; - return g_blockSize; + prefs->blockSizeId = bsid; + prefs->blockSize = blockSizeTable[prefs->blockSizeId-minBlockSizeID]; + return prefs->blockSize; } -size_t LZ4IO_setBlockSize(size_t blockSize) +size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize) { static const size_t minBlockSize = 32; static const size_t maxBlockSize = 4 MB; unsigned bsid = 0; if (blockSize < minBlockSize) blockSize = minBlockSize; if (blockSize > maxBlockSize) blockSize = maxBlockSize; - g_blockSize = blockSize; + prefs->blockSize = blockSize; blockSize--; /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */ while (blockSize >>= 2) bsid++; if (bsid < 7) bsid = 7; - g_blockSizeId = bsid-3; - return g_blockSize; + prefs->blockSizeId = bsid-3; + return prefs->blockSize; } -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode) +int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode) { - g_blockIndependence = (blockMode == LZ4IO_blockIndependent); - return g_blockIndependence; + prefs->blockIndependence = (blockMode == LZ4IO_blockIndependent); + return prefs->blockIndependence; } /* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int enable) +int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int enable) { - g_blockChecksum = (enable != 0); - return g_blockChecksum; + prefs->blockChecksum = (enable != 0); + return prefs->blockChecksum; } /* Default setting : checksum enabled */ -int LZ4IO_setStreamChecksumMode(int enable) +int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int enable) { - g_streamChecksum = (enable != 0); - return g_streamChecksum; + prefs->streamChecksum = (enable != 0); + return prefs->streamChecksum; } /* Default setting : 0 (no notification) */ @@ -228,27 +268,29 @@ int LZ4IO_setNotificationLevel(int level) } /* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable) +int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable) { - g_sparseFileSupport = (enable!=0); - return g_sparseFileSupport; + prefs->sparseFileSupport = (enable!=0); + return prefs->sparseFileSupport; } /* Default setting : 0 (disabled) */ -int LZ4IO_setContentSize(int enable) +int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable) { - g_contentSizeFlag = (enable!=0); - return g_contentSizeFlag; + prefs->contentSizeFlag = (enable!=0); + return prefs->contentSizeFlag; } /* Default setting : 0 (disabled) */ -void LZ4IO_favorDecSpeed(int favor) +void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor) { - g_favorDecSpeed = (favor!=0); + prefs->favorDecSpeed = (favor!=0); } -static U32 g_removeSrcFile = 0; -void LZ4IO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag) +{ + prefs->removeSrcFile = (flag>0); +} @@ -283,7 +325,7 @@ static FILE* LZ4IO_openSrcFile(const char* srcFileName) /** FIO_openDstFile() : * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LZ4IO_openDstFile(const char* dstFileName) +static FILE* LZ4IO_openDstFile(LZ4IO_prefs_t* const prefs, const char* dstFileName) { FILE* f; @@ -291,12 +333,12 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4,"Using stdout for output\n"); f = stdout; SET_BINARY_MODE(stdout); - if (g_sparseFileSupport==1) { - g_sparseFileSupport = 0; + if (prefs->sparseFileSupport==1) { + prefs->sparseFileSupport = 0; DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); } } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + if (!prefs->overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); if (f != NULL) { /* dest exists, prompt for overwrite authorization */ fclose(f); @@ -317,7 +359,7 @@ static FILE* LZ4IO_openDstFile(const char* dstFileName) } /* sparse file */ - if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } + if (f && prefs->sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } return f; } @@ -347,7 +389,7 @@ static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSi /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ -int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) +int LZ4IO_compressFilename_Legacy(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel) { typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC; @@ -365,7 +407,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output if (finput == NULL) EXM_THROW(20, "%s : open file error ", input_filename); - foutput = LZ4IO_openDstFile(output_filename); + foutput = LZ4IO_openDstFile(prefs, output_filename); if (foutput == NULL) { fclose(finput); EXM_THROW(20, "%s : open file error ", input_filename); @@ -444,7 +486,7 @@ typedef struct { LZ4F_CDict* cdict; } cRess_t; -static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { +static void* LZ4IO_createDict(LZ4IO_prefs_t* const prefs, size_t *dictSize) { size_t readSize; size_t dictEnd = 0; size_t dictLen = 0; @@ -452,6 +494,7 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { size_t circularBufSize = LZ4_MAX_DICT_SIZE; char* circularBuf; char* dictBuf; + const char* dictFilename = prefs->dictionaryFilename; FILE* dictFile; if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided"); @@ -501,23 +544,23 @@ static void* LZ4IO_createDict(const char* dictFilename, size_t *dictSize) { return dictBuf; } -static LZ4F_CDict* LZ4IO_createCDict(void) { +static LZ4F_CDict* LZ4IO_createCDict(LZ4IO_prefs_t* const prefs) { size_t dictionarySize; void* dictionaryBuffer; LZ4F_CDict* cdict; - if (!g_useDictionary) { + if (!prefs->useDictionary) { return NULL; } - dictionaryBuffer = LZ4IO_createDict(g_dictionaryFilename, &dictionarySize); + dictionaryBuffer = LZ4IO_createDict(prefs, &dictionarySize); if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize); free(dictionaryBuffer); return cdict; } -static cRess_t LZ4IO_createCResources(void) +static cRess_t LZ4IO_createCResources(LZ4IO_prefs_t* const prefs) { - const size_t blockSize = g_blockSize; + const size_t blockSize = prefs->blockSize; cRess_t ress; LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); @@ -530,7 +573,7 @@ static cRess_t LZ4IO_createCResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); - ress.cdict = LZ4IO_createCDict(); + ress.cdict = LZ4IO_createCDict(prefs); return ress; } @@ -552,7 +595,7 @@ static void LZ4IO_freeCResources(cRess_t ress) * result : 0 : compression completed correctly * 1 : missing or pb opening srcFileName */ -static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) +static int LZ4IO_compressFilename_extRess(LZ4IO_prefs_t* const io_prefs, cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; @@ -561,7 +604,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, void* const srcBuffer = ress.srcBuffer; void* const dstBuffer = ress.dstBuffer; const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = g_blockSize; + const size_t blockSize = io_prefs->blockSize; size_t readSize; LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; @@ -569,7 +612,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Init */ srcFile = LZ4IO_openSrcFile(srcFileName); if (srcFile == NULL) return 1; - dstFile = LZ4IO_openDstFile(dstFileName); + dstFile = LZ4IO_openDstFile(io_prefs, dstFileName); if (dstFile == NULL) { fclose(srcFile); return 1; } memset(&prefs, 0, sizeof(prefs)); @@ -577,12 +620,12 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; - prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum; - prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; - prefs.favorDecSpeed = g_favorDecSpeed; - if (g_contentSizeFlag) { + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)io_prefs->blockIndependence; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)io_prefs->blockSizeId; + prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)io_prefs->blockChecksum; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum; + prefs.favorDecSpeed = io_prefs->favorDecSpeed; + if (io_prefs->contentSizeFlag) { U64 const fileSize = UTIL_getFileSize(srcFileName); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ if (fileSize==0) @@ -661,7 +704,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, UTIL_setFileStat(dstFileName, &statbuf); } } - if (g_removeSrcFile) { /* remove source file : --rm */ + if (io_prefs->removeSrcFile) { /* remove source file : --rm */ if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } @@ -676,13 +719,13 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, } -int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +int LZ4IO_compressFilename(LZ4IO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName, int compressionLevel) { UTIL_time_t const timeStart = UTIL_getTime(); clock_t const cpuStart = clock(); - cRess_t const ress = LZ4IO_createCResources(); + cRess_t const ress = LZ4IO_createCResources(prefs); - int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + int const result = LZ4IO_compressFilename_extRess(prefs, ress, srcFileName, dstFileName, compressionLevel); /* Free resources */ LZ4IO_freeCResources(ress); @@ -701,7 +744,7 @@ int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int #define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) +int LZ4IO_compressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) { int i; int missed_files = 0; @@ -711,7 +754,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, cRess_t ress; if (dstFileName == NULL) return ifntSize; /* not enough memory */ - ress = LZ4IO_createCResources(); + ress = LZ4IO_createCResources(prefs); /* loop on each file */ for (i=0; i<ifntSize; i++) { @@ -720,7 +763,7 @@ int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, strcpy(dstFileName, inFileNamesTable[i]); strcat(dstFileName, suffix); - missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); + missed_files += LZ4IO_compressFilename_extRess(prefs, ress, inFileNamesTable[i], dstFileName, compressionLevel); } /* Close & Free */ @@ -746,7 +789,7 @@ static unsigned LZ4IO_readLE32 (const void* s) } -static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned LZ4IO_fwriteSparse(LZ4IO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) { const size_t sizeT = sizeof(size_t); const size_t maskT = sizeT -1 ; @@ -756,7 +799,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer const size_t* const bufferTEnd = bufferT + bufferSizeT; const size_t segmentSizeT = (32 KB) / sizeT; - if (!g_sparseFileSupport) { /* normal write */ + if (!prefs->sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); return 0; @@ -825,7 +868,7 @@ static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) static unsigned g_magicRead = 0; /* out-parameter of LZ4IO_decodeLegacyStream() */ -static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) +static unsigned long long LZ4IO_decodeLegacyStream(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput) { unsigned long long streamSize = 0; unsigned storedSkips = 0; @@ -859,7 +902,7 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !"); streamSize += decodeSize; /* Write Block */ - storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); /* success or die */ + storedSkips = LZ4IO_fwriteSparse(prefs, foutput, out_buff, decodeSize, storedSkips); /* success or die */ } } if (ferror(finput)) EXM_THROW(54, "Read error : ferror"); @@ -885,19 +928,19 @@ typedef struct { size_t dictBufferSize; } dRess_t; -static void LZ4IO_loadDDict(dRess_t* ress) { - if (!g_useDictionary) { +static void LZ4IO_loadDDict(LZ4IO_prefs_t* const prefs, dRess_t* ress) { + if (!prefs->useDictionary) { ress->dictBuffer = NULL; ress->dictBufferSize = 0; return; } - ress->dictBuffer = LZ4IO_createDict(g_dictionaryFilename, &ress->dictBufferSize); + ress->dictBuffer = LZ4IO_createDict(prefs, &ress->dictBufferSize); if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary"); } static const size_t LZ4IO_dBufferSize = 64 KB; -static dRess_t LZ4IO_createDResources(void) +static dRess_t LZ4IO_createDResources(LZ4IO_prefs_t* const prefs) { dRess_t ress; @@ -912,7 +955,7 @@ static dRess_t LZ4IO_createDResources(void) ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - LZ4IO_loadDDict(&ress); + LZ4IO_loadDDict(prefs, &ress); ress.dstFile = NULL; return ress; @@ -928,7 +971,7 @@ static void LZ4IO_freeDResources(dRess_t ress) } -static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) +static unsigned long long LZ4IO_decompressLZ4F(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, FILE* dstFile) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; @@ -963,8 +1006,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* Write Block */ if (decodedBytes) { - if (!g_testMode) - storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); + if (!prefs->testMode) + storedSkips = LZ4IO_fwriteSparse(prefs, dstFile, ress.dstBuffer, decodedBytes, storedSkips); filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); } @@ -975,7 +1018,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE /* can be out because readSize == 0, which could be an fread() error */ if (ferror(srcFile)) EXM_THROW(67, "Read error"); - if (!g_testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); + if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips); if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); return filesize; @@ -984,7 +1027,7 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE #define PTSIZE (64 KB) #define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) +static unsigned long long LZ4IO_passThrough(LZ4IO_prefs_t* const prefs, FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) { size_t buffer[PTSIZET]; size_t readBytes = 1; @@ -997,7 +1040,7 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne while (readBytes) { readBytes = fread(buffer, 1, PTSIZE, finput); total += readBytes; - storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, storedSkips); + storedSkips = LZ4IO_fwriteSparse(prefs, foutput, buffer, readBytes, storedSkips); } if (ferror(finput)) EXM_THROW(51, "Read Error"); @@ -1024,7 +1067,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) +static unsigned long long selectDecoder(LZ4IO_prefs_t* const prefs, dRess_t ress, FILE* finput, FILE* foutput) { unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber; @@ -1050,10 +1093,10 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return LZ4IO_decompressLZ4F(ress, finput, foutput); + return LZ4IO_decompressLZ4F(prefs, ress, finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return LZ4IO_decodeLegacyStream(finput, foutput); + return LZ4IO_decodeLegacyStream(prefs, finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); @@ -1070,9 +1113,9 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu default: if (nbFrames == 1) { /* just started */ /* Wrong magic number at the beginning of 1st stream */ - if (!g_testMode && g_overwrite) { + if (!prefs->testMode && prefs->overwrite && prefs->passThrough) { nbFrames = 0; - return LZ4IO_passThrough(finput, foutput, MNstore); + return LZ4IO_passThrough(prefs, finput, foutput, MNstore); } EXM_THROW(44,"Unrecognized header : file cannot be decoded"); } @@ -1087,7 +1130,7 @@ static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutpu } -static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int LZ4IO_decompressSrcFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) { FILE* const foutput = ress.dstFile; unsigned long long filesize = 0; @@ -1099,14 +1142,14 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con /* Loop over multiple streams */ for ( ; ; ) { /* endless loop, see break condition */ unsigned long long const decodedSize = - selectDecoder(ress, finput, foutput); + selectDecoder(prefs, ress, finput, foutput); if (decodedSize == ENDOFSTREAM) break; filesize += decodedSize; } /* Close input */ fclose(finput); - if (g_removeSrcFile) { /* --rm */ + if (prefs->removeSrcFile) { /* --rm */ if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } @@ -1120,11 +1163,11 @@ static int LZ4IO_decompressSrcFile(dRess_t ress, const char* input_filename, con } -static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) +static int LZ4IO_decompressDstFile(LZ4IO_prefs_t* const prefs, dRess_t ress, const char* input_filename, const char* output_filename) { stat_t statbuf; int stat_result = 0; - FILE* const foutput = LZ4IO_openDstFile(output_filename); + FILE* const foutput = LZ4IO_openDstFile(prefs, output_filename); if (foutput==NULL) return 1; /* failure */ if ( strcmp(input_filename, stdinmark) @@ -1132,7 +1175,7 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con stat_result = 1; ress.dstFile = foutput; - LZ4IO_decompressSrcFile(ress, input_filename, output_filename); + LZ4IO_decompressSrcFile(prefs, ress, input_filename, output_filename); fclose(foutput); @@ -1148,12 +1191,12 @@ static int LZ4IO_decompressDstFile(dRess_t ress, const char* input_filename, con } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +int LZ4IO_decompressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename) { - dRess_t const ress = LZ4IO_createDResources(); + dRess_t const ress = LZ4IO_createDResources(prefs); clock_t const start = clock(); - int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename); + int const missingFiles = LZ4IO_decompressDstFile(prefs, ress, input_filename, output_filename); clock_t const end = clock(); double const seconds = (double)(end - start) / CLOCKS_PER_SEC; @@ -1164,7 +1207,7 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file } -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix) { int i; int skippedFiles = 0; @@ -1172,16 +1215,16 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz char* outFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; size_t const suffixSize = strlen(suffix); - dRess_t ress = LZ4IO_createDResources(); + dRess_t ress = LZ4IO_createDResources(prefs); if (outFileName==NULL) return ifntSize; /* not enough memory */ - ress.dstFile = LZ4IO_openDstFile(stdoutmark); + ress.dstFile = LZ4IO_openDstFile(prefs, stdoutmark); for (i=0; i<ifntSize; i++) { size_t const ifnSize = strlen(inFileNamesTable[i]); const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize; if (!strcmp(suffix, stdoutmark)) { - missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark); + missingFiles += LZ4IO_decompressSrcFile(prefs, ress, inFileNamesTable[i], stdoutmark); continue; } if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); if (outFileName==NULL) return ifntSize; } @@ -1192,7 +1235,7 @@ int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSiz } memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); outFileName[ifnSize-suffixSize] = '\0'; - missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName); + missingFiles += LZ4IO_decompressDstFile(prefs, ress, inFileNamesTable[i], outFileName); } LZ4IO_freeDResources(ress); diff --git a/programs/lz4io.h b/programs/lz4io.h index 33de41f..0c28784 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -48,65 +48,77 @@ static const char nulmark[] = "nul"; static const char nulmark[] = "/dev/null"; #endif +/* ************************************************** */ +/* ****************** Type Definitions ************** */ +/* ************************************************** */ + +typedef struct LZ4IO_prefs_s LZ4IO_prefs_t; + +LZ4IO_prefs_t* LZ4IO_defaultPreferences(void); +void LZ4IO_freePreferences(LZ4IO_prefs_t* const prefs); /* ************************************************** */ /* ****************** Functions ********************* */ /* ************************************************** */ -int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); +int LZ4IO_compressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename, int compressionlevel); +int LZ4IO_decompressFilename(LZ4IO_prefs_t* const prefs, const char* input_filename, const char* output_filename); -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); -int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); +int LZ4IO_compressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_decompressMultipleFilenames(LZ4IO_prefs_t* const prefs, const char** inFileNamesTable, int ifntSize, const char* suffix); /* ************************************************** */ /* ****************** Parameters ******************** */ /* ************************************************** */ -int LZ4IO_setDictionaryFilename(const char* dictionaryFilename); +int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename); + +/* Default setting : passThrough = 0; + return : passThrough mode (0/1) */ +int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes); /* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LZ4IO_setOverwrite(int yes); +int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes); /* Default setting : testMode = 0; return : testMode (0/1) */ -int LZ4IO_setTestMode(int yes); +int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes); /* blockSizeID : valid values : 4-5-6-7 return : 0 if error, blockSize if OK */ -size_t LZ4IO_setBlockSizeID(unsigned blockSizeID); +size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned blockSizeID); /* blockSize : valid values : 32 -> 4MB return : 0 if error, actual blocksize if OK */ -size_t LZ4IO_setBlockSize(size_t blockSize); +size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize); /* Default setting : independent blocks */ typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; -int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode); +int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode); /* Default setting : no block checksum */ -int LZ4IO_setBlockChecksumMode(int xxhash); +int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); /* Default setting : stream checksum enabled */ -int LZ4IO_setStreamChecksumMode(int xxhash); +int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); /* Default setting : 0 (no notification) */ int LZ4IO_setNotificationLevel(int level); /* Default setting : 0 (disabled) */ -int LZ4IO_setSparseFile(int enable); +int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable); /* Default setting : 0 == no content size present in frame header */ -int LZ4IO_setContentSize(int enable); +int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable); /* Default setting : 0 == src file preserved */ -void LZ4IO_setRemoveSrcFile(unsigned flag); +void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag); /* Default setting : 0 == favor compression ratio * Note : 1 only works for high compression levels (10+) */ -void LZ4IO_favorDecSpeed(int favor); +void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor); #endif /* LZ4IO_H_237902873 */ diff --git a/tests/.gitignore b/tests/.gitignore index 9aa42a0..c4f9092 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,6 +10,7 @@ fuzzer32 fasttest roundTripTest checkTag +checkFrame # test artefacts tmp* diff --git a/tests/Makefile b/tests/Makefile index 8dcef6d..7d49b31 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -284,6 +284,11 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat test "$(shell ./datagen -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell ./datagen -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1 ! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0 ! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1 + # Test for #596 + @echo "TEST" > tmp-tlb-test + $(LZ4) tmp-tlb-test + $(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2 + $(DIFF) -q tmp-tlb-test tmp-tlb-test2 @$(RM) tmp-tlb* @@ -332,8 +337,11 @@ test-lz4-testmode: lz4 datagen ! ./datagen | $(LZ4) -t ! ./datagen | $(LZ4) -tf @echo "\n ---- pass-through mode ----" - ! ./datagen | $(LZ4) -d > $(VOID) - ./datagen | $(LZ4) -df > $(VOID) + @echo "Why hello there " > tmp-tlt2.lz4 + ! $(LZ4) -f tmp-tlt2.lz4 > $(VOID) + ! ./datagen | $(LZ4) -dc > $(VOID) + ! ./datagen | $(LZ4) -df > $(VOID) + ./datagen | $(LZ4) -dcf > $(VOID) @echo "Hello World !" > tmp-tlt1 $(LZ4) -dcf tmp-tlt1 @echo "from underground..." > tmp-tlt2 @@ -342,7 +350,7 @@ test-lz4-testmode: lz4 datagen ! $(LZ4) file-does-not-exist ! $(LZ4) -f file-does-not-exist ! $(LZ4) -fm file1-dne file2-dne - @$(RM) tmp-tlt + @$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4 test-lz4-opt-parser: lz4 datagen @echo "\n ---- test opt-parser ----" diff --git a/tests/frametest.c b/tests/frametest.c index b93f4fe..14906a0 100644 --- a/tests/frametest.c +++ b/tests/frametest.c @@ -658,6 +658,29 @@ int basicTests(U32 seed, double compressibility) CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; } + DISPLAYLEVEL(3, "getBlockSize test: \n"); + { size_t result; + unsigned blockSizeID; + for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) { + result = LZ4F_getBlockSize(blockSizeID); + CHECK(result); + DISPLAYLEVEL(3, "Returned block size of %zu bytes for blockID %u \n", + result, blockSizeID); + } + + /* Test an invalid input that's too large */ + result = LZ4F_getBlockSize(8); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + + /* Test an invalid input that's too small */ + result = LZ4F_getBlockSize(3); + if(!LZ4F_isError(result) || + LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) + goto _output_error; + } + DISPLAYLEVEL(3, "Skippable frame test : \n"); { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fab1fce..893ffc1 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -48,6 +48,7 @@ #include <string.h> /* strcmp */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ #include <assert.h> +#include <limits.h> /* INT_MAX */ #if defined(__unix__) && defined(_AIX) # include <sys/mman.h> /* mmap */ #endif @@ -1064,45 +1065,81 @@ static void FUZ_unitTests(int compressionLevel) } /* LZ4 HC streaming tests */ - { LZ4_streamHC_t* sp; - LZ4_streamHC_t sHC; + { LZ4_streamHC_t sHC; /* statically allocated */ U64 crcOrig; int result; /* Allocation test */ - sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); + DISPLAYLEVEL(3, " Basic HC allocation : "); + { LZ4_streamHC_t* const sp = LZ4_createStreamHC(); + FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); + LZ4_freeStreamHC(sp); + } + DISPLAYLEVEL(3, " OK \n"); /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_resetStreamHC(&sHC, compressionLevel); - result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + DISPLAYLEVEL(3, " Simple HC round-trip : "); + { U64 const crc64 = XXH64(testInput, testCompressedSize, 0); + LZ4_resetStreamHC(&sHC, compressionLevel); + result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); + + /* long sequence test */ + DISPLAYLEVEL(3, " Long sequence HC test : "); + { size_t const blockSize = 1 MB; + size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ + void* const block = malloc(blockSize); + void* const dstBlock = malloc(targetSize+1); + BYTE const sentinel = 101; + int srcSize; + + assert(block != NULL); assert(dstBlock != NULL); + memset(block, 0, blockSize); + ((char*)dstBlock)[targetSize] = sentinel; + + LZ4_resetStreamHC(&sHC, 3); + assert(blockSize < INT_MAX); + srcSize = (int)blockSize; + assert(targetSize < INT_MAX); + result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3); + DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize); + FUZ_CHECKTEST(result!=4116, "LZ4_compress_HC_destSize() : compression must fill dstBuffer completely, but no more !"); + FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, "LZ4_compress_HC_destSize()") + + LZ4_resetStreamHC(&sHC, 3); /* make sure the context is clean after the test */ + free(block); + free(dstBlock); + } + DISPLAYLEVEL(3, " OK \n"); /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); } + DISPLAYLEVEL(3, " HC dictionary compression test : "); + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 64 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); + } } + DISPLAYLEVEL(3, " OK \n"); /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); + U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0); LZ4_resetStreamHC_fast(&sHC, compressionLevel); LZ4_loadDictHC(&sHC, testInput, segSize); result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); @@ -1116,22 +1153,23 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - } + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); + } } /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); } + { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); + LZ4_resetStreamHC_fast(&sHC, compressionLevel); + LZ4_loadDictHC(&sHC, testInput, 32 KB); + result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result); + FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); + + result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); + } } /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; diff --git a/visual/VS2017/lz4.sln b/visual/VS2017/lz4.sln index 78f223b..72e98fc 100644 --- a/visual/VS2017/lz4.sln +++ b/visual/VS2017/lz4.sln @@ -1,7 +1,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}" -EndProject +# Visual Studio 15 +VisualStudioVersion = 15.0.28307.271 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4-dll", "liblz4-dll\liblz4-dll.vcxproj", "{9800039D-4AAA-43A4-BB78-FEF6F4836927}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblz4", "liblz4\liblz4.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}" @@ -27,14 +27,6 @@ Global Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64 {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.ActiveCfg = Debug|Win32 {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|Win32.Build.0 = Debug|Win32 {9800039D-4AAA-43A4-BB78-FEF6F4836927}.Debug|x64.ActiveCfg = Debug|x64 @@ -95,4 +87,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {BBC259B2-BABF-47CD-8A6A-7B8318A803AC} + EndGlobalSection EndGlobal |