From de52a52518ad0a76a0a078d3db4ee440fad2ff7a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 13:06:51 -0700 Subject: added LZ4_streamHC_t init test which includes an alignment test --- tests/fuzzer.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 6c2ccb7..820ac8e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1200,6 +1200,25 @@ static void FUZ_unitTests(int compressionLevel) } } + DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : "); + { struct { + LZ4_streamHC_t hc1; + char c1; + LZ4_streamHC_t hc2; + char c2; + LZ4_streamHC_t hc3; + } shc; + DISPLAYLEVEL(3, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", + &(shc.hc1), &(shc.hc2), &(shc.hc3), (unsigned)sizeof(LZ4_streamHC_t)); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc1), sizeof(shc.hc1)) == NULL, "hc1 (%p) failed init", &(shc.hc1) ); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc2), sizeof(shc.hc2)) == NULL, "hc2 (%p) failed init", &(shc.hc2) ); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc3), sizeof(shc.hc3)) == NULL, "hc3 (%p) failed init", &(shc.hc3) ); + FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc.hc1) + 1, sizeof(shc.hc1)) != NULL, + "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc.hc1) + 1 ); + /* no need to release anything : LZ4_streamHC_t is a simple POD type */ + } + DISPLAYLEVEL(3, "OK \n"); + /* LZ4 HC streaming tests */ { LZ4_streamHC_t sHC; /* statically allocated */ int result; @@ -1211,7 +1230,7 @@ static void FUZ_unitTests(int compressionLevel) FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); LZ4_freeStreamHC(sp); } - DISPLAYLEVEL(3, " OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* simple HC compression test */ DISPLAYLEVEL(3, "Simple HC round-trip : "); @@ -1226,7 +1245,7 @@ static void FUZ_unitTests(int compressionLevel) { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption"); } } - DISPLAYLEVEL(3, " OK \n"); + DISPLAYLEVEL(3, "OK \n"); /* long sequence test */ DISPLAYLEVEL(3, "Long sequence HC_destSize test : "); -- cgit v0.12 From 58c0a549adef617aa92124353d8a5a76094df17c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 14:10:00 -0700 Subject: fix cppcheck unused variable warning --- tests/fuzzer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 820ac8e..4fef4af 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1208,6 +1208,7 @@ static void FUZ_unitTests(int compressionLevel) char c2; LZ4_streamHC_t hc3; } shc; + (void)shc.c1; (void)shc.c2; /* tell cppcheck these variables are unused */ DISPLAYLEVEL(3, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", &(shc.hc1), &(shc.hc2), &(shc.hc3), (unsigned)sizeof(LZ4_streamHC_t)); FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc1), sizeof(shc.hc1)) == NULL, "hc1 (%p) failed init", &(shc.hc1) ); -- cgit v0.12 From c661adda10529f9b4d4fa14ea7e5c68ec66ccaa4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 15:43:46 -0700 Subject: fixed x32 test on Travis --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6074f08..2281394 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,18 +47,15 @@ matrix: - make clean travis-install - make clean clangtest - - # 14.04 LTS Server Edition 64 bit - - name: (Trusty) i386 gcc test - dist: trusty + - name: x32 compatibility test addons: apt: packages: - - libc6-dev-i386 - gcc-multilib script: - make -C tests test MOREFLAGS=-mx32 + # 14.04 LTS Server Edition 64 bit # presume clang >= v3.9.0 - name: (Trusty) USan test dist: trusty -- cgit v0.12 From 2631002f74631145ee7385932035487fb08227f3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 15:51:14 -0700 Subject: preserver alignment test on Visual Studio x64 this it works fine in this environment (only x86 is suspicious) --- lib/lz4hc.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index cd802d8..cf03eba 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -53,7 +53,7 @@ #include "lz4hc.h" -/*=== Common LZ4 definitions ===*/ +/*=== Common definitions ===*/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" #endif @@ -61,19 +61,29 @@ # pragma clang diagnostic ignored "-Wunused-function" #endif -/*=== Enums ===*/ -typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; - - #define LZ4_COMMONDEFS_ONLY #ifndef LZ4_SRC_INCLUDED #include "lz4.c" /* LZ4_count, constants, mem */ #endif + +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + /*=== Constants ===*/ #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) #define LZ4_OPT_NUM (1<<12) +/* for some reason, Visual Studio fails the aligment test on 32-bit x86 : + * it reports an aligment of 8-bytes, + * while LZ4_streamHC_t only requires alignment of 4-bytes + * resulting in initialization error when allocating state with malloc() */ +#if (defined(_MSC_VER) && !defined(_M_X64)) +# define LZ4_ALIGN_TEST 0 +#else +# define LZ4_ALIGN_TEST 1 +#endif /*=== Macros ===*/ #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) @@ -161,8 +171,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) { size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; - if (bitsToRotate == 0) - return pattern; + if (bitsToRotate == 0) return pattern; return LZ4HC_rotl32(pattern, (int)bitsToRotate); } @@ -912,9 +921,7 @@ LZ4HC_compress_generic ( int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ +#if LZ4_ALIGN_TEST static size_t LZ4_streamHC_t_alignment(void) { typedef struct { char c; LZ4_streamHC_t t; } t_a; @@ -927,9 +934,7 @@ static size_t LZ4_streamHC_t_alignment(void) int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ +#if LZ4_ALIGN_TEST assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */ #endif if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ @@ -1003,9 +1008,7 @@ LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; if (buffer == NULL) return NULL; if (size < sizeof(LZ4_streamHC_t)) return NULL; -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while actually aligning LZ4_streamHC_t on 4 bytes. */ +#if LZ4_ALIGN_TEST if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */ #endif /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ -- cgit v0.12 From 389eacdfeaefee5326de2352ff42aa0195f156a0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 16:37:34 -0700 Subject: Appveyor: added compilation and runtime fuzzer tests to all Windows compiler targets --- appveyor.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 57b6dda..31894d3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,10 +47,14 @@ build_script: make -v && echo ----- && if not [%PLATFORM%]==[clang] ( - make -C programs lz4 && make -C tests fullbench && make -C lib lib + make -C programs lz4 && + make -C tests fullbench && + make -C tests fuzzer && + make -C lib lib ) ELSE ( make -C programs lz4 CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && make -C tests fullbench CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && + make -C tests fuzzer CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && make -C lib lib CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" ) ) @@ -77,7 +81,7 @@ build_script: 7z.exe a bin\lz4_x86.zip NEWS .\bin\lz4.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include && appveyor PushArtifact bin\lz4_x86.zip ) - - if [%COMPILER%]==[gcc] (COPY tests\fullbench.exe programs\) + - if [%COMPILER%]==[gcc] (COPY tests\*.exe programs\) - if [%COMPILER%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && @@ -110,7 +114,9 @@ test_script: lz4 -i1b10 lz4.exe && lz4 -i1b15 lz4.exe && echo ------- lz4 tested ------- && - fullbench.exe -i1 fullbench.exe + fullbench.exe -i1 fullbench.exe && + echo trying to launch fuzzer.exe && + fuzzer.exe -v -T30s ) artifacts: -- cgit v0.12 From 4b97866333523b2bc778d88fe68b6b86ea26ce6e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 2 Oct 2020 18:02:41 -0700 Subject: reduce new test stack usage use heap instead --- tests/fuzzer.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 4fef4af..3eb5789 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1201,22 +1201,24 @@ static void FUZ_unitTests(int compressionLevel) } DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : "); - { struct { + { typedef struct { LZ4_streamHC_t hc1; char c1; LZ4_streamHC_t hc2; char c2; LZ4_streamHC_t hc3; - } shc; - (void)shc.c1; (void)shc.c2; /* tell cppcheck these variables are unused */ + } shct; + shct* const shc = malloc(sizeof(*shc)); + assert(shc != NULL); + memset(shc, 0, sizeof(*shc)); DISPLAYLEVEL(3, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", - &(shc.hc1), &(shc.hc2), &(shc.hc3), (unsigned)sizeof(LZ4_streamHC_t)); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc1), sizeof(shc.hc1)) == NULL, "hc1 (%p) failed init", &(shc.hc1) ); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc2), sizeof(shc.hc2)) == NULL, "hc2 (%p) failed init", &(shc.hc2) ); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc.hc3), sizeof(shc.hc3)) == NULL, "hc3 (%p) failed init", &(shc.hc3) ); - FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc.hc1) + 1, sizeof(shc.hc1)) != NULL, - "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc.hc1) + 1 ); - /* no need to release anything : LZ4_streamHC_t is a simple POD type */ + &(shc->hc1), &(shc->hc2), &(shc->hc3), (unsigned)sizeof(LZ4_streamHC_t)); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", &(shc->hc1) ); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", &(shc->hc2) ); + FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", &(shc->hc3) ); + FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL, + "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->hc1) + 1 ); + free(shc); } DISPLAYLEVEL(3, "OK \n"); -- cgit v0.12 From ad552101ddeef3bf1967048ecb3cc5844a1ad470 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 31 Oct 2020 01:55:04 -0700 Subject: fix minor explicit cast --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 3eb5789..84f9693 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1208,7 +1208,7 @@ static void FUZ_unitTests(int compressionLevel) char c2; LZ4_streamHC_t hc3; } shct; - shct* const shc = malloc(sizeof(*shc)); + shct* const shc = (shct*)malloc(sizeof(*shc)); assert(shc != NULL); memset(shc, 0, sizeof(*shc)); DISPLAYLEVEL(3, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", -- cgit v0.12 From e968a241293fe86c0f8b115a1223d9d78a0eda00 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Nov 2020 14:46:48 -0800 Subject: unified alignment test across lz4.c and lz4hc.c --- lib/lz4.c | 33 +++++++++++++++++++------- lib/lz4hc.c | 24 +++++-------------- tests/fuzzer.c | 74 ++++++++++++++++++++++++++++++++++++---------------------- 3 files changed, 76 insertions(+), 55 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 0290ea2..9e6abba 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -178,6 +178,20 @@ #define unlikely(expr) expect((expr) != 0, 0) #endif +/* for some reason, Visual Studio can fail the aligment test on 32-bit x86 : + * it sometimes report an aligment of 8-bytes (at least in some configurations), + * while only providing a `malloc()` memory area aligned on 4-bytes, + * which is inconsistent with malloc() contract. + * The source of the issue is still unclear. + * Mitigation : made the alignment test optional */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# if (defined(_MSC_VER) && !defined(_M_X64)) +# define LZ4_ALIGN_TEST 0 /* disable on win32+visual */ +# else +# define LZ4_ALIGN_TEST 1 +# endif +#endif + /*-************************************ * Memory routines @@ -243,6 +257,11 @@ static const int LZ4_minLength = (MFLIMIT+1); # define DEBUGLOG(l, ...) {} /* disabled */ #endif +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + /*-************************************ * Types @@ -1406,26 +1425,22 @@ LZ4_stream_t* LZ4_createStream(void) return lz4s; } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - it reports an aligment of 8-bytes, - while actually aligning LZ4_stream_t on 4 bytes. */ static size_t LZ4_stream_t_alignment(void) { +#if LZ4_ALIGN_TEST typedef struct { char c; LZ4_stream_t t; } t_a; return sizeof(t_a) - sizeof(LZ4_stream_t); -} +#else + return 1; /* effectively disabled */ #endif +} LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) { DEBUGLOG(5, "LZ4_initStream"); if (buffer == NULL) { return NULL; } if (size < sizeof(LZ4_stream_t)) { return NULL; } -#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 : - it reports an aligment of 8-bytes, - while actually aligning LZ4_stream_t on 4 bytes. */ - if (((size_t)buffer) & (LZ4_stream_t_alignment() - 1)) { return NULL; } /* alignment check */ -#endif + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); return (LZ4_stream_t*)buffer; } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index cf03eba..29b6073 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -75,15 +75,6 @@ typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) #define LZ4_OPT_NUM (1<<12) -/* for some reason, Visual Studio fails the aligment test on 32-bit x86 : - * it reports an aligment of 8-bytes, - * while LZ4_streamHC_t only requires alignment of 4-bytes - * resulting in initialization error when allocating state with malloc() */ -#if (defined(_MSC_VER) && !defined(_M_X64)) -# define LZ4_ALIGN_TEST 0 -#else -# define LZ4_ALIGN_TEST 1 -#endif /*=== Macros ===*/ #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) @@ -921,23 +912,22 @@ LZ4HC_compress_generic ( int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } -#if LZ4_ALIGN_TEST static size_t LZ4_streamHC_t_alignment(void) { +#if LZ4_ALIGN_TEST typedef struct { char c; LZ4_streamHC_t t; } t_a; return sizeof(t_a) - sizeof(LZ4_streamHC_t); -} +#else + return 1; /* effectively disabled */ #endif +} /* state is presumed correctly initialized, * in which case its size and alignment have already been validate */ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; -#if LZ4_ALIGN_TEST - assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */ -#endif - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0; LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); LZ4HC_init_internal (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) @@ -1008,9 +998,7 @@ LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; if (buffer == NULL) return NULL; if (size < sizeof(LZ4_streamHC_t)) return NULL; -#if LZ4_ALIGN_TEST - if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */ -#endif + if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE); DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 84f9693..29c6a8a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1125,27 +1125,47 @@ static void FUZ_unitTests(int compressionLevel) } } } } } DISPLAYLEVEL(3, " OK \n"); - /* LZ4 streaming tests */ - { LZ4_stream_t streamingState; - U64 crcOrig; - int result; + DISPLAYLEVEL(3, "LZ4_initStream with multiple valid alignments : "); + { typedef struct { + LZ4_stream_t state1; + LZ4_stream_t state2; + char c; + LZ4_stream_t state3; + } shct; + shct* const shc = (shct*)malloc(sizeof(*shc)); + assert(shc != NULL); + memset(shc, 0, sizeof(*shc)); + DISPLAYLEVEL(3, "state1(%p) state2(%p) state3(%p) size(0x%x): ", + &(shc->state1), &(shc->state2), &(shc->state3), (unsigned)sizeof(LZ4_stream_t)); + FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", &(shc->state1) ); + FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", &(shc->state2) ); + FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", &(shc->state3) ); + FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL, + "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->state1) + 1 ); + free(shc); + } + DISPLAYLEVEL(3, "all inits OK \n"); - /* Allocation test */ - { LZ4_stream_t* const statePtr = LZ4_createStream(); - FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); - LZ4_freeStream(statePtr); - } + /* Allocation test */ + { LZ4_stream_t* const statePtr = LZ4_createStream(); + FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); + LZ4_freeStream(statePtr); + } - /* simple compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - LZ4_initStream(&streamingState, sizeof(streamingState)); - result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); - FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!"); + /* LZ4 streaming tests */ + { LZ4_stream_t streamingState; - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); } + /* simple compression test */ + { U64 const crcOrig = XXH64(testInput, testCompressedSize, 0); + LZ4_initStream(&streamingState, sizeof(streamingState)); + { int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); + FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!"); + { int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize); + FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); + } } + { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + } } /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -1167,7 +1187,7 @@ static void FUZ_unitTests(int compressionLevel) LZ4_setStreamDecode(&decodeStateFast, NULL, 0); while (iNext + messageSize < testCompressedSize) { - int compressedSize; + int compressedSize; U64 crcOrig; XXH64_update(&xxhOrig, testInput + iNext, messageSize); crcOrig = XXH64_digest(&xxhOrig); @@ -1175,15 +1195,15 @@ static void FUZ_unitTests(int compressionLevel) compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1); FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed"); - result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); + { int const r = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize); + FUZ_CHECKTEST(r!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); } XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewSafe); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); } - result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); - FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); + { int const r = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize); + FUZ_CHECKTEST(r!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); } XXH64_update(&xxhNewFast, testVerify + dNext, messageSize); { U64 const crcNew = XXH64_digest(&xxhNewFast); @@ -1196,16 +1216,14 @@ static void FUZ_unitTests(int compressionLevel) messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; if (rNext + messageSize > ringBufferSize) rNext = 0; if (dNext + messageSize > dBufferSize) dNext = 0; - } - } + } } } DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : "); { typedef struct { LZ4_streamHC_t hc1; - char c1; LZ4_streamHC_t hc2; - char c2; + char c; LZ4_streamHC_t hc3; } shct; shct* const shc = (shct*)malloc(sizeof(*shc)); @@ -1220,7 +1238,7 @@ static void FUZ_unitTests(int compressionLevel) "hc1+1 (%p) init must fail, due to bad alignment", (char*)&(shc->hc1) + 1 ); free(shc); } - DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "all inits OK \n"); /* LZ4 HC streaming tests */ { LZ4_streamHC_t sHC; /* statically allocated */ -- cgit v0.12 From 78433070abd9da36f225a5be1c302e42cdad67bd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Nov 2020 14:48:43 -0800 Subject: document LZ4_ALIGN_TEST --- lib/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/README.md b/lib/README.md index 3653c81..7a9e4fd 100644 --- a/lib/README.md +++ b/lib/README.md @@ -77,6 +77,9 @@ The following build macro can be selected to adjust source code behavior at comp In most cases, it's not expected to be necessary, but it can be legitimately considered for less common platforms. +- `LZ4_ALIGN_TEST` : disable state alignment test when set to 0. + Is generally enabled by default, except on win32+visual. + #### Amalgamation -- cgit v0.12 From 0c56f838edfb1eb06553985a141e5243f740ef27 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Nov 2020 16:15:51 -0800 Subject: unified internal state declaration align on `void*` instead : there is no `long long` inside the structure --- lib/lz4.h | 60 ++++++++++++++++++++++++++---------------------------------- lib/lz4hc.h | 55 +++++++++++++++---------------------------------------- 2 files changed, 41 insertions(+), 74 deletions(-) diff --git a/lib/lz4.h b/lib/lz4.h index b11275e..78c2542 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -573,68 +573,60 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const ************************************************************** * Do not use these definitions directly. * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Accessing members will expose code to API and/or ABI break in future versions of the library. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; -struct LZ4_stream_t_internal { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t tableType; - const uint8_t* dictionary; - const LZ4_stream_t_internal* dictCtx; - uint32_t dictSize; -}; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; #else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; struct LZ4_stream_t_internal { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int tableType; - const unsigned char* dictionary; + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + const LZ4_byte* dictionary; const LZ4_stream_t_internal* dictCtx; - unsigned int dictSize; + LZ4_u32 dictSize; }; typedef struct { - const unsigned char* externalDict; - const unsigned char* prefixEnd; + const LZ4_byte* externalDict; size_t extDictSize; + const LZ4_byte* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; -#endif /*! LZ4_stream_t : - * information structure to track an LZ4 stream. + * Do not use below internal definitions directly ! + * Declare or allocate an LZ4_stream_t instead. * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. * The structure definition can be convenient for static allocation * (on stack, or as part of larger structure). * Init this structure with LZ4_initStream() before first use. * note : only use this definition in association with static linking ! - * this definition is not API/ABI safe, and may change in a future version. + * this definition is not API/ABI safe, and may change in future versions. */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ ) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMSIZE_VOIDP ((sizeof(LZ4_stream_t_internal) + sizeof(void*)-1) / sizeof(void*)) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_VOIDP * sizeof(void*)) union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; + void* table[LZ4_STREAMSIZE_VOIDP]; LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ +}; /* previously typedef'd to LZ4_stream_t */ + /*! LZ4_initStream() : v1.9.0+ * An LZ4_stream_t structure must be initialized at least once. diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 44e35bb..228bb11 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -198,57 +198,32 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; -struct LZ4HC_CCtx_internal -{ - uint32_t hashTable[LZ4HC_HASHTABLESIZE]; - uint16_t chainTable[LZ4HC_MAXD]; - const uint8_t* end; /* next block here to continue on current prefix */ - const uint8_t* base; /* All index relative to this position */ - const uint8_t* dictBase; /* alternate base for extDict */ - uint32_t dictLimit; /* below that point, need extDict */ - uint32_t lowLimit; /* below that point, no more dict */ - uint32_t nextToUpdate; /* index from which to continue dictionary update */ - short compressionLevel; - int8_t favorDecSpeed; /* favor decompression speed if this flag set, - otherwise, favor compression ratio */ - int8_t dirty; /* stream has to be fully reset if this flag is set */ - const LZ4HC_CCtx_internal* dictCtx; -}; - -#else - typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; struct LZ4HC_CCtx_internal { - unsigned int hashTable[LZ4HC_HASHTABLESIZE]; - unsigned short chainTable[LZ4HC_MAXD]; - const unsigned char* end; /* next block here to continue on current prefix */ - const unsigned char* base; /* All index relative to this position */ - const unsigned char* dictBase; /* alternate base for extDict */ - unsigned int dictLimit; /* below that point, need extDict */ - unsigned int lowLimit; /* below that point, no more dict */ - unsigned int nextToUpdate; /* index from which to continue dictionary update */ - short compressionLevel; - char favorDecSpeed; /* favor decompression speed if this flag set, - otherwise, favor compression ratio */ - char dirty; /* stream has to be fully reset if this flag is set */ + LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE]; + LZ4_u16 chainTable[LZ4HC_MAXD]; + const LZ4_byte* end; /* next block here to continue on current prefix */ + const LZ4_byte* base; /* All index relative to this position */ + const LZ4_byte* dictBase; /* alternate base for extDict */ + LZ4_u32 dictLimit; /* below that point, need extDict */ + LZ4_u32 lowLimit; /* below that point, no more dict */ + LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */ + short compressionLevel; + LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set, + otherwise, favor compression ratio */ + LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */ const LZ4HC_CCtx_internal* dictCtx; }; -#endif - /* Do not use these definitions directly ! * Declare or allocate an LZ4_streamHC_t instead. */ -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/ -#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +#define LZ4_STREAMHCSIZE_VOIDP ((sizeof(LZ4HC_CCtx_internal) + sizeof(void*)-1) / sizeof(void*)) +#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_VOIDP * sizeof(void*)) union LZ4_streamHC_u { - size_t table[LZ4_STREAMHCSIZE_SIZET]; + void* table[LZ4_STREAMHCSIZE_VOIDP]; LZ4HC_CCtx_internal internal_donotuse; }; /* previously typedef'd to LZ4_streamHC_t */ -- cgit v0.12 From 211d653ff866ee98a51ab135067369f52f6bdc7b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Nov 2020 16:43:14 -0800 Subject: re-enable alignment test on all targets --- lib/README.md | 5 +++-- lib/lz4.c | 20 ++++++-------------- lib/lz4.h | 2 +- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/README.md b/lib/README.md index 7a9e4fd..ee5d3ba 100644 --- a/lib/README.md +++ b/lib/README.md @@ -77,8 +77,9 @@ The following build macro can be selected to adjust source code behavior at comp In most cases, it's not expected to be necessary, but it can be legitimately considered for less common platforms. -- `LZ4_ALIGN_TEST` : disable state alignment test when set to 0. - Is generally enabled by default, except on win32+visual. +- `LZ4_ALIGN_TEST` : alignment test ensures that the memory area + passed as argument to become a compression state is suitable aligned. + This test can be disabled, if it proves flaky, by setting this value to 0. #### Amalgamation diff --git a/lib/lz4.c b/lib/lz4.c index 9e6abba..c902654 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -178,18 +178,10 @@ #define unlikely(expr) expect((expr) != 0, 0) #endif -/* for some reason, Visual Studio can fail the aligment test on 32-bit x86 : - * it sometimes report an aligment of 8-bytes (at least in some configurations), - * while only providing a `malloc()` memory area aligned on 4-bytes, - * which is inconsistent with malloc() contract. - * The source of the issue is still unclear. - * Mitigation : made the alignment test optional */ +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ #ifndef LZ4_ALIGN_TEST /* can be externally provided */ -# if (defined(_MSC_VER) && !defined(_M_X64)) -# define LZ4_ALIGN_TEST 0 /* disable on win32+visual */ -# else -# define LZ4_ALIGN_TEST 1 -# endif +# define LZ4_ALIGN_TEST 1 #endif @@ -476,7 +468,7 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si switch(offset) { case 1: - memset(v, *srcPtr, 8); + MEM_INIT(v, *srcPtr, 8); break; case 2: LZ4_memcpy(v, srcPtr, 2); @@ -1441,7 +1433,7 @@ LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) if (buffer == NULL) { return NULL; } if (size < sizeof(LZ4_stream_t)) { return NULL; } if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; - MEM_INIT(buffer, 0, sizeof(LZ4_stream_t)); + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); return (LZ4_stream_t*)buffer; } @@ -1450,7 +1442,7 @@ LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); } void LZ4_resetStream_fast(LZ4_stream_t* ctx) { diff --git a/lib/lz4.h b/lib/lz4.h index 78c2542..026279b 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -618,7 +618,7 @@ typedef struct { * (on stack, or as part of larger structure). * Init this structure with LZ4_initStream() before first use. * note : only use this definition in association with static linking ! - * this definition is not API/ABI safe, and may change in future versions. + * this definition is not API/ABI safe, and may change in future versions. */ #define LZ4_STREAMSIZE_VOIDP ((sizeof(LZ4_stream_t_internal) + sizeof(void*)-1) / sizeof(void*)) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_VOIDP * sizeof(void*)) -- cgit v0.12 From 67e661a2ad39258bcb5c6fcc31c9239ff3aabc71 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Nov 2020 18:26:13 -0800 Subject: static state size for better inter-version compatibility --- lib/lz4.h | 4 ++-- lib/lz4hc.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lz4.h b/lib/lz4.h index 026279b..c4324af 100644 --- a/lib/lz4.h +++ b/lib/lz4.h @@ -620,8 +620,8 @@ typedef struct { * note : only use this definition in association with static linking ! * this definition is not API/ABI safe, and may change in future versions. */ -#define LZ4_STREAMSIZE_VOIDP ((sizeof(LZ4_stream_t_internal) + sizeof(void*)-1) / sizeof(void*)) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_VOIDP * sizeof(void*)) +#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */ +#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*)) union LZ4_stream_u { void* table[LZ4_STREAMSIZE_VOIDP]; LZ4_stream_t_internal internal_donotuse; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 228bb11..3d441fb 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -220,8 +220,8 @@ struct LZ4HC_CCtx_internal /* Do not use these definitions directly ! * Declare or allocate an LZ4_streamHC_t instead. */ -#define LZ4_STREAMHCSIZE_VOIDP ((sizeof(LZ4HC_CCtx_internal) + sizeof(void*)-1) / sizeof(void*)) -#define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_VOIDP * sizeof(void*)) +#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */ +#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*)) union LZ4_streamHC_u { void* table[LZ4_STREAMHCSIZE_VOIDP]; LZ4HC_CCtx_internal internal_donotuse; -- cgit v0.12