diff options
| -rw-r--r-- | tests/.gitignore | 3 | ||||
| -rw-r--r-- | tests/Makefile | 9 | ||||
| -rw-r--r-- | tests/abiTest.c | 216 | ||||
| -rw-r--r-- | tests/test-lz4-abi.py | 150 |
4 files changed, 377 insertions, 1 deletions
diff --git a/tests/.gitignore b/tests/.gitignore index 346c989..5337fdb 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -13,9 +13,12 @@ checkTag checkFrame decompress-partial decompress-partial-usingDict +abiTest + # test artefacts tmp* versionsTest +abiTests lz4_all.c # local tests diff --git a/tests/Makefile b/tests/Makefile index b67f135..6f063a1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -127,7 +127,8 @@ clean: fasttest$(EXT) roundTripTest$(EXT) \ datagen$(EXT) checkTag$(EXT) \ frameTest$(EXT) decompress-partial$(EXT) \ - lz4_all.c + abiTest$(EXT) \ + lz4_all.c @$(RM) -rf $(TESTDIR) @echo Cleaning completed @@ -139,6 +140,12 @@ versionsTest: listTest: lz4 QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py +abiTest: LDLIBS += -llz4 + +.PHONY: abiTests +abiTests: + $(PYTHON) test-lz4-abi.py + checkTag: checkTag.c $(LZ4DIR)/lz4.h $(CC) $(FLAGS) $< -o $@$(EXT) diff --git a/tests/abiTest.c b/tests/abiTest.c new file mode 100644 index 0000000..c637511 --- /dev/null +++ b/tests/abiTest.c @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree), + * meaning you may select, at your option, one of the above-listed licenses. + */ + +/* + * abiTest : + * ensure ABI stability expectations are not broken by a new version +**/ + + +/*=========================================== +* Dependencies +*==========================================*/ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free, exit */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ +#include <assert.h> +#include <sys/types.h> /* stat */ +#include <sys/stat.h> /* stat */ +#include "xxhash.h" + +#include "lz4.h" +#include "lz4frame.h" + + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define MSG(...) fprintf(stderr, __VA_ARGS__) + +#define CONTROL_MSG(c, ...) { \ + if ((c)) { \ + MSG(__VA_ARGS__); \ + MSG(" \n"); \ + abort(); \ + } \ +} + + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* const ip1 = (const char*)buff1; + const char* const ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos<buffSize; pos++) + if (ip1[pos]!=ip2[pos]) + break; + + return pos; +} + + +LZ4_stream_t LZ4_cState; +LZ4_streamDecode_t LZ4_dState; + +/** roundTripTest() : + * Compresses `srcBuff` into `compressedBuff`, + * then decompresses `compressedBuff` into `resultBuff`. + * If clevel==0, compression level is derived from srcBuff's content head bytes. + * This function abort() if it detects any round-trip error. + * Therefore, if it returns, round trip is considered successfully validated. + * Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)` + * for compression to be guaranteed to work */ +static void roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcSize) +{ + int const acceleration = 1; + // Note : can't use LZ4_initStream(), because it's only present since v1.9.0 + memset(&LZ4_cState, 0, sizeof(LZ4_cState)); + { int const cSize = LZ4_compress_fast_continue(&LZ4_cState, (const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, acceleration); + CONTROL_MSG(cSize == 0, "Compression error !"); + { int const dInit = LZ4_setStreamDecode(&LZ4_dState, NULL, 0); + CONTROL_MSG(dInit == 0, "LZ4_setStreamDecode error !"); + } + { int const dSize = LZ4_decompress_safe_continue (&LZ4_dState, (const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity); + CONTROL_MSG(dSize < 0, "Decompression detected an error !"); + CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !"); + } } + + /* check potential content corruption error */ + assert(resultBuffCapacity >= srcSize); + { size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize); + CONTROL_MSG(errorPos != srcSize, + "Silent decoding corruption, at pos %u !!!", + (unsigned)errorPos); + } + +} + +static void roundTripCheck(const void* srcBuff, size_t srcSize) +{ + size_t const cBuffSize = LZ4_COMPRESSBOUND(srcSize); + void* const cBuff = malloc(cBuffSize); + void* const rBuff = malloc(cBuffSize); + + if (!cBuff || !rBuff) { + fprintf(stderr, "not enough memory ! \n"); + exit(1); + } + + roundTripTest(rBuff, cBuffSize, + cBuff, cBuffSize, + srcBuff, srcSize); + + free(rBuff); + free(cBuff); +} + + +static size_t getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (size_t)statbuf.st_size; +} + + +static int isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +/** loadFile() : + * requirement : `buffer` size >= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + MSG("Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + MSG("Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + MSG("Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName) +{ + size_t const fileSize = getFileSize(fileName); + void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); + if (!buffer) { + MSG("not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize); + free (buffer); +} + + +int bad_usage(const char* exeName) +{ + MSG(" \n"); + MSG("bad usage: \n"); + MSG(" \n"); + MSG("%s [Options] fileName \n", exeName); + MSG(" \n"); + MSG("Options: \n"); + MSG("-# : use #=[0-9] compression level (default:0 == random) \n"); + return 1; +} + + +int main(int argCount, const char** argv) +{ + const char* const exeName = argv[0]; + int argNb = 1; + MSG("starting abiTest: \n"); + + assert(argCount >= 1); + if (argCount < 2) return bad_usage(exeName); + + if (argNb >= argCount) return bad_usage(exeName); + + fileCheck(argv[argNb]); + MSG("no pb detected \n"); + return 0; +} diff --git a/tests/test-lz4-abi.py b/tests/test-lz4-abi.py new file mode 100644 index 0000000..378f62f --- /dev/null +++ b/tests/test-lz4-abi.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Test LZ4 interoperability between versions""" + +# +# Copyright (C) 2011-present, Takayuki Matsuoka +# All rights reserved. +# GPL v2 License +# + +import glob +import subprocess +import filecmp +import os +import shutil +import sys +import hashlib + +repo_url = 'https://github.com/lz4/lz4.git' +tmp_dir_name = 'tests/abiTests' +env_flags = ' ' # '-j MOREFLAGS="-g -O0 -fsanitize=address"' +make_cmd = 'make' +git_cmd = 'git' +test_dat_src = ['README.md'] +head = 'v999' + +def proc(cmd_args, pipe=True, env=False): + if env == False: + env = os.environ.copy() + # we want the address sanitizer for abi tests + env["MOREFLAGS"] = "-fsanitize=address" + if pipe: + subproc = subprocess.Popen(cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env = env) + else: + subproc = subprocess.Popen(cmd_args, env = env) + return subproc.communicate() + +def make(args, pipe=True, env=False): + return proc([make_cmd] + ['-j'] + ['V=1'] + args, pipe, env) + +def git(args, pipe=True): + return proc([git_cmd] + args, pipe) + +def get_git_tags(): + # Only start from first v1.7.x format release + stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]']) + tags = stdout.decode('utf-8').split() + return tags + +# https://stackoverflow.com/a/19711609/2132223 +def sha1_of_file(filepath): + with open(filepath, 'rb') as f: + return hashlib.sha1(f.read()).hexdigest() + +if __name__ == '__main__': + error_code = 0 + base_dir = os.getcwd() + '/..' # /path/to/lz4 + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest + clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4 + lib_dir = base_dir + '/lib' # /path/to/lz4/lib + test_dir = base_dir + '/tests' + os.makedirs(tmp_dir, exist_ok=True) + + # since Travis clones limited depth, we should clone full repository + if not os.path.isdir(clone_dir): + git(['clone', repo_url, clone_dir]) + + # Retrieve all release tags + print('Retrieve all release tags :') + os.chdir(clone_dir) + tags = [head] + get_git_tags() + print(tags); + + # Build all versions of liblz4 + # note : naming scheme only works on Linux + for tag in tags: + print('building library ', tag) + os.chdir(base_dir) +# if not os.path.isfile(dst_liblz4) or tag == head: + if tag != head: + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG> + #print('r_dir = ', r_dir) # for debug + os.makedirs(r_dir, exist_ok=True) + os.chdir(clone_dir) + git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.']) + os.chdir(r_dir + '/lib') # /path/to/lz4/lz4test/<TAG>/lib + else: + # print('lib_dir = {}', lib_dir) # for debug + os.chdir(lib_dir) # for debug + make(['clean']) + make(['liblz4']) + + print(' ') + print('******************************') + print('Round trip expecting current ABI but linking to older Dynamic Library version') + print('******************************') + os.chdir(test_dir) + # Start with matching version : should be no problem + build_env = os.environ.copy() + build_env["LDFLAGS"] = "-L../lib" + build_env["LDLIBS"] = "-llz4" + # we use asan to detect any out-of-bound read or write + build_env["MOREFLAGS"] = "-fsanitize=address" + os.remove('abiTest') + make(['abiTest'], env=build_env) + proc(['./abiTest'] + ['README.md']) + + for tag in tags: + print('linking to lib tag = ', tag) + run_env = os.environ.copy() + run_env["LD_LIBRARY_PATH"] = 'abiTests/{}/lib'.format(tag) + # check we are linking to the right library version at run time + proc(['ldd'] + ['./abiTest'], pipe=False, env=run_env) + # now run with mismatched library version + proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env) + + print(' ') + print('******************************') + print('Round trip using current Dynamic Library expecting older ABI version') + print('******************************') + + for tag in tags: + print('building using older lib ', tag) + build_env = os.environ.copy() + if tag != head: + build_env["CPPFLAGS"] = '-IabiTests/{}/lib'.format(tag) + build_env["LDFLAGS"] = '-LabiTests/{}/lib'.format(tag) + else: + build_env["CPPFLAGS"] = '-I../lib' + build_env["LDFLAGS"] = '-L../lib' + build_env["LDLIBS"] = "-llz4" + build_env["MOREFLAGS"] = "-fsanitize=address" + os.remove('abiTest') + make(['abiTest'], pipe=False, env=build_env) + + print('run with CURRENT library version (head)') + run_env = os.environ.copy() + run_env["LD_LIBRARY_PATH"] = '../lib' + # check we are linking to the right library version at run time + proc(['ldd'] + ['./abiTest'], pipe=False, env=run_env) + # now run with mismatched library version + proc(['./abiTest'] + test_dat_src, pipe=False, env=run_env) + + + if error_code != 0: + print('ERROR') + + sys.exit(error_code) |