summaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
authorYann Collet <Cyan4973@users.noreply.github.com>2019-08-20 17:35:58 (GMT)
committerGitHub <noreply@github.com>2019-08-20 17:35:58 (GMT)
commit3d67671559be723b0912bbee2fcd2eb14783a721 (patch)
tree8c4c3cacf90599887a5b02a46ec6f052f4422ef0 /tests
parent398e36c756a3067de8e2b35dd380baef040dfe0d (diff)
parentfdf2ef5809ca875c454510610764d9125ef2ebbd (diff)
downloadlz4-3d67671559be723b0912bbee2fcd2eb14783a721.zip
lz4-3d67671559be723b0912bbee2fcd2eb14783a721.tar.gz
lz4-3d67671559be723b0912bbee2fcd2eb14783a721.tar.bz2
Merge pull request #780 from lz4/dev
v1.9.2
Diffstat (limited to 'tests')
-rw-r--r--tests/.gitignore1
-rw-r--r--tests/Makefile28
-rw-r--r--tests/frametest.c297
-rw-r--r--tests/fullbench.c97
-rw-r--r--tests/fuzzer.c321
-rw-r--r--tests/test-lz4-list.py282
6 files changed, 785 insertions, 241 deletions
diff --git a/tests/.gitignore b/tests/.gitignore
index c4f9092..0d13df8 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -15,6 +15,7 @@ checkFrame
# test artefacts
tmp*
versionsTest
+lz4_all.c
# local tests
afl
diff --git a/tests/Makefile b/tests/Makefile
index 67514e4..422baba 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -114,7 +114,7 @@ clean:
frametest$(EXT) frametest32$(EXT) \
fasttest$(EXT) roundTripTest$(EXT) \
datagen$(EXT) checkTag$(EXT) \
- frameTest$(EXT)
+ frameTest$(EXT) lz4_all.c
@$(RM) -rf $(TESTDIR)
@echo Cleaning completed
@@ -122,10 +122,13 @@ clean:
versionsTest:
$(PYTHON) test-lz4-versions.py
+.PHONY: listTest
+listTest: lz4
+ QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py
+
checkTag: checkTag.c $(LZ4DIR)/lz4.h
$(CC) $(FLAGS) $< -o $@$(EXT)
-
#-----------------------------------------------------------------------------
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
@@ -150,19 +153,19 @@ list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: test
-test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation
+test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest
.PHONY: test32
test32: CFLAGS+=-m32
test32: test
-.PHONY: test-amalgamation
-test-amalgamation: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c
- cat $(LZ4DIR)/lz4.c > lz4_all.c
- cat $(LZ4DIR)/lz4hc.c >> lz4_all.c
- cat $(LZ4DIR)/lz4frame.c >> lz4_all.c
- $(CC) -I$(LZ4DIR) -c lz4_all.c
- $(RM) lz4_all.c
+test-amalgamation: lz4_all.o
+
+lz4_all.o: lz4_all.c
+ $(CC) $(CFLAGS) $(CPPFLAGS) -c $^ -o $@
+
+lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
+ cat $^ > $@
test-install: lz4 lib liblz4.pc
lz4_root=.. ./test_install.sh
@@ -373,6 +376,7 @@ test-lz4-testmode: lz4 datagen
@echo "\n ---- non-existing source ----"
! $(LZ4) file-does-not-exist
! $(LZ4) -f file-does-not-exist
+ ! $(LZ4) -t file-does-not-exist
! $(LZ4) -fm file1-dne file2-dne
@$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4
@@ -444,7 +448,7 @@ test-fuzzer32: CFLAGS += -m32
test-fuzzer32: test-fuzzer
test-frametest: frametest
- ./frametest $(FUZZER_TIME)
+ ./frametest -v $(FUZZER_TIME)
test-frametest32: CFLAGS += -m32
test-frametest32: test-frametest
@@ -463,6 +467,8 @@ test-mem: lz4 datagen fuzzer frametest fullbench
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
+ valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4
+ valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4
$(RM) ftm*
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
diff --git a/tests/frametest.c b/tests/frametest.c
index bf95beb..1b932e4 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -41,11 +41,12 @@
#include <string.h> /* strcmp */
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
#include <assert.h>
-#include "lz4frame.h" /* include multiple times to test correctness/safety */
+#include "lz4frame.h" /* included multiple times to test correctness/safety */
#include "lz4frame.h"
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
#include "lz4frame.h"
+#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
#include "lz4.h" /* LZ4_VERSION_STRING */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
@@ -150,8 +151,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
size_t const length = MIN(lengthRand, bufferSize - pos);
size_t const end = pos + length;
while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
- }
- }
+ } }
}
@@ -541,7 +541,7 @@ int basicTests(U32 seed, double compressibility)
cdict, NULL) );
DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
(unsigned)dictSize, (unsigned)cSizeWithDict);
- if (cSizeWithDict >= cSizeNoDict) goto _output_error; /* must be more efficient */
+ if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */
crcOrig = XXH64(CNBuffer, dictSize, 0);
DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
@@ -703,8 +703,8 @@ int basicTests(U32 seed, double compressibility)
while (ip < iend) {
unsigned nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
- size_t oSize = oend-op;
- if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+ size_t oSize = (size_t)(oend-op);
+ if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
@@ -722,8 +722,8 @@ int basicTests(U32 seed, double compressibility)
while (ip < iend) {
unsigned const nbBits = FUZ_rand(&randState) % maxBits;
size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
- size_t oSize = oend-op;
- if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+ size_t oSize = (size_t)(oend-op);
+ if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
@@ -739,7 +739,7 @@ int basicTests(U32 seed, double compressibility)
while (ip < iend) {
size_t iSize = 10;
size_t oSize = 10;
- if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+ if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
op += oSize;
ip += iSize;
@@ -763,55 +763,173 @@ _output_error:
}
-static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
+typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
+
+static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
{
- size_t p=0;
- const BYTE* b1=(const BYTE*)buff1;
- const BYTE* b2=(const BYTE*)buff2;
- DISPLAY("locateBuffDiff: looking for error position \n");
- if (nonContiguous) {
- DISPLAY("mode %u: non-contiguous output (%u bytes), cannot search \n",
- nonContiguous, (unsigned)size);
- return;
+ if (displayLevel >= 5) {
+ size_t p=0;
+ const BYTE* b1=(const BYTE*)buff1;
+ const BYTE* b2=(const BYTE*)buff2;
+ DISPLAY("locateBuffDiff: looking for error position \n");
+ if (o_scenario != o_contiguous) {
+ DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
+ (int)o_scenario, (unsigned)size);
+ return;
+ }
+ while (p < size && b1[p]==b2[p]) p++;
+ if (p != size) {
+ DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+ }
}
- while (p < size && b1[p]==b2[p]) p++;
- if (p != size) {
- DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
+}
+
+# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
+ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); }
+# undef CHECK
+# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
+
+
+size_t test_lz4f_decompression_wBuffers(
+ const void* cSrc, size_t cSize,
+ void* dst, size_t dstCapacity, o_scenario_e o_scenario,
+ const void* srcRef, size_t decompressedSize,
+ U64 crcOrig,
+ U32* const randState,
+ LZ4F_dctx* const dCtx,
+ U32 seed, U32 testNb)
+{
+ const BYTE* ip = (const BYTE*)cSrc;
+ const BYTE* const iend = ip + cSize;
+
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + dstCapacity;
+
+ unsigned const suggestedBits = FUZ_highbit((U32)cSize);
+ unsigned const maxBits = MAX(3, suggestedBits);
+ size_t totalOut = 0;
+ size_t moreToFlush = 0;
+ XXH64_state_t xxh64;
+ XXH64_reset(&xxh64, 1);
+ assert(ip < iend);
+ while (ip < iend) {
+ unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
+ unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1;
+ size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1;
+ size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip));
+ size_t iSize = iSizeMax;
+ size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2;
+ size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op));
+ size_t oSize = oSizeMax;
+ BYTE const mark = (BYTE)(FUZ_rand(randState) & 255);
+ LZ4F_decompressOptions_t dOptions;
+ memset(&dOptions, 0, sizeof(dOptions));
+ dOptions.stableDst = FUZ_rand(randState) & 1;
+ if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
+ if (op + oSizeMax < oend) op[oSizeMax] = mark;
+
+ DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
+
+ /* read data from byte-exact buffer to catch out-of-bound reads */
+ { void* const iBuffer = malloc(iSizeMax);
+ assert(iBuffer != NULL);
+ memcpy(iBuffer, ip, iSizeMax);
+ moreToFlush = LZ4F_decompress(dCtx, op, &oSize, iBuffer, &iSize, &dOptions);
+ free(iBuffer);
+ }
+ DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
+
+ if (op + oSizeMax < oend) {
+ CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
+ "Decompression overwrites beyond assigned dst size",
+ op[oSizeMax], mark);
+ }
+ if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid)
+ locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
+ if (LZ4F_isError(moreToFlush)) return moreToFlush;
+
+ XXH64_update(&xxh64, op, oSize);
+ totalOut += oSize;
+ op += oSize;
+ ip += iSize;
+ if (o_scenario == o_noncontiguous) {
+ if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */
+ op++; /* create a gap between consecutive output */
+ }
+ if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */
+ if ( (op == oend) /* no more room for output; can happen with bogus input */
+ && (iSize == 0)) /* no input consumed */
+ break;
}
+ if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
+ if (totalOut) { /* otherwise, it's a skippable frame */
+ U64 const crcDecoded = XXH64_digest(&xxh64);
+ if (crcDecoded != crcOrig) {
+ locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
+ return LZ4F_ERROR_contentChecksum_invalid;
+ } }
+ return 0;
+}
+
+
+size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
+ const void* srcRef, size_t decompressedSize,
+ U64 crcOrig,
+ U32* const randState,
+ LZ4F_dctx* const dCtx,
+ U32 seed, U32 testNb)
+{
+ o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
+ /* tighten dst buffer conditions */
+ size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
+ (decompressedSize * 2) + 128 :
+ decompressedSize;
+ size_t result;
+ void* const dstBuffer = malloc(dstCapacity);
+ assert(dstBuffer != NULL);
+
+ result = test_lz4f_decompression_wBuffers(cSrc, cSize,
+ dstBuffer, dstCapacity, o_scenario,
+ srcRef, decompressedSize,
+ crcOrig,
+ randState,
+ dCtx,
+ seed, testNb);
+
+ free(dstBuffer);
+ return result;
}
int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s)
{
- int testResult = 0;
unsigned testNb = 0;
- size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
- void* srcBuffer = NULL;
- size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL) + 4 MB; /* needs some margin */
+ size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
+ void* CNBuffer = NULL;
+ size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */
void* compressedBuffer = NULL;
void* decodedBuffer = NULL;
U32 coreRand = seed;
LZ4F_decompressionContext_t dCtx = NULL;
+ LZ4F_decompressionContext_t dCtxNoise = NULL;
LZ4F_compressionContext_t cCtx = NULL;
clock_t const startClock = clock();
clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
-# undef CHECK
-# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
- DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
-# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
/* Create buffers */
{ size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
+ { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
+ CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
{ size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
- srcBuffer = malloc(srcDataLength);
- CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
+ CNBuffer = malloc(CNBufferLength);
+ CHECK(CNBuffer==NULL, "CNBuffer Allocation failed");
compressedBuffer = malloc(compressedBufferSize);
CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
- decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
+ decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
- FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
+ FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand);
/* jump to requested testNb */
for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */
@@ -819,10 +937,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
/* main fuzzer test loop */
for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
U32 randState = coreRand ^ prime1;
- unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1;
+ unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1;
size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1;
- size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize);
- const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId;
+ size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
+ const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
U64 const crcOrig = XXH64(srcStart, srcSize, 1);
LZ4F_preferences_t prefs;
@@ -848,9 +966,11 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
FUZ_writeLE32(op+4, (U32)srcSize);
cSize = srcSize+8;
+
} else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */
cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr);
CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
+
} else { /* multi-segments compression */
const BYTE* ip = srcStart;
const BYTE* const iend = srcStart + srcSize;
@@ -913,58 +1033,53 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize);
}
+
/* multi-segments decompression */
- { const BYTE* ip = (const BYTE*)compressedBuffer;
- const BYTE* const iend = ip + cSize;
- BYTE* op = (BYTE*)decodedBuffer;
- BYTE* const oend = op + srcDataLength;
- unsigned const suggestedBits = FUZ_highbit((U32)cSize);
- unsigned const maxBits = MAX(3, suggestedBits);
- unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
- size_t totalOut = 0;
- size_t decSize = 0;
- XXH64_state_t xxh64;
- XXH64_reset(&xxh64, 1);
- assert(ip < iend);
- while (ip < iend) {
- unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
- unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
- size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
- size_t iSize = MIN(iSizeMax, (size_t)(iend-ip));
- size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
- size_t oSize = MIN(oSizeMax, (size_t)(oend-op));
- LZ4F_decompressOptions_t dOptions;
- memset(&dOptions, 0, sizeof(dOptions));
- dOptions.stableDst = FUZ_rand(&randState) & 1;
- if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */
- decSize = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
- if (LZ4F_getErrorCode(decSize) == LZ4F_ERROR_contentChecksum_invalid)
- locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
- CHECK(LZ4F_isError(decSize), "Decompression failed (error %i:%s)",
- (int)decSize, LZ4F_getErrorName(decSize));
- XXH64_update(&xxh64, op, (U32)oSize);
- totalOut += oSize;
- op += oSize;
- ip += iSize;
- op += nonContiguousDst;
- if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */
- }
- CHECK(decSize != 0, "Frame decompression failed (error %i)", (int)decSize);
- if (totalOut) { /* otherwise, it's a skippable frame */
- U64 const crcDecoded = XXH64_digest(&xxh64);
- if (crcDecoded != crcOrig) {
- locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
- EXIT_MSG("Decompression corruption");
- } }
+ DISPLAYLEVEL(6, "normal decompression \n");
+ { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb);
+ CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)",
+ (int)result, LZ4F_getErrorName(result));
}
- }
+
+#if 1
+ /* insert noise into src */
+ { U32 const maxNbBits = FUZ_highbit((U32)cSize);
+ size_t pos = 0;
+ for (;;) {
+ /* keep some original src */
+ { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
+ size_t const mask = (1<<nbBits) - 1;
+ size_t const skipLength = FUZ_rand(&randState) & mask;
+ pos += skipLength;
+ }
+ if (pos >= cSize) break;
+ /* add noise */
+ { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
+ U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
+ size_t const mask = (1<<nbBits) - 1;
+ size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
+ size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
+ size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength);
+ memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength);
+ pos += noiseLength;
+ } } }
+
+ /* test decompression on noisy src */
+ DISPLAYLEVEL(6, "noisy decompression \n");
+ test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb);
+ /* note : we don't analyze result here : it probably failed, which is expected.
+ * We just check for potential out-of-bound reads and writes. */
+ LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */
+#endif
+
+} /* for ( ; (testNb < nbTests) ; ) */
DISPLAYLEVEL(2, "\rAll tests completed \n");
-_end:
LZ4F_freeDecompressionContext(dCtx);
+ LZ4F_freeDecompressionContext(dCtxNoise);
LZ4F_freeCompressionContext(cCtx);
- free(srcBuffer);
+ free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
@@ -972,11 +1087,7 @@ _end:
DISPLAY("press enter to finish \n");
(void)getchar();
}
- return testResult;
-
-_output_error:
- testResult = 1;
- goto _end;
+ return 0;
}
@@ -1002,8 +1113,8 @@ int main(int argc, const char** argv)
U32 seed=0;
int seedset=0;
int argNb;
- int nbTests = nbTestsDefault;
- int testNb = 0;
+ unsigned nbTests = nbTestsDefault;
+ unsigned testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 duration=0;
@@ -1048,7 +1159,7 @@ int main(int argc, const char** argv)
nbTests=0; duration=0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
- nbTests += *argument - '0';
+ nbTests += (unsigned)(*argument - '0');
argument++;
}
break;
@@ -1071,7 +1182,7 @@ int main(int argc, const char** argv)
case '6':
case '7':
case '8':
- case '9': duration *= 10; duration += *argument++ - '0'; continue;
+ case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
}
break;
}
@@ -1083,7 +1194,7 @@ int main(int argc, const char** argv)
seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
- seed += *argument - '0';
+ seed += (U32)(*argument - '0');
argument++;
}
break;
@@ -1092,7 +1203,7 @@ int main(int argc, const char** argv)
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
- testNb += *argument - '0';
+ testNb += (unsigned)(*argument - '0');
argument++;
}
break;
@@ -1126,7 +1237,7 @@ int main(int argc, const char** argv)
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
- if (nbTests<=0) nbTests=1;
+ nbTests += (nbTests==0); /* avoid zero */
if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
if (result) return 1;
diff --git a/tests/fullbench.c b/tests/fullbench.c
index d2af662..7d74d3f 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -343,11 +343,44 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS
assert(inSize >= 0);
assert(outSize >= 0);
result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL);
- if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame\n"); exit(8); }
- if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); }
+ if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); }
+ if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); }
return (int)dstSize;
}
+static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize)
+{
+ size_t totalInSize = (size_t)srcSize;
+ size_t maxOutSize = (size_t)dstSize;
+
+ size_t inPos = 0;
+ size_t inSize = 0;
+ size_t outPos = 0;
+ size_t outRemaining = maxOutSize - outPos;
+
+ for (;;) {
+ size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
+ assert(!LZ4F_isError(sizeHint));
+
+ inPos += inSize;
+ inSize = sizeHint;
+
+ outPos += outRemaining;
+ outRemaining = maxOutSize - outPos;
+
+ if (!sizeHint) break;
+ }
+
+ /* frame completed */
+ if (inPos != totalInSize) {
+ DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
+ (unsigned)inPos, (unsigned)totalInSize);
+ exit(10);
+ }
+ return (int)outPos;
+
+}
+
#define NB_COMPRESSION_ALGORITHMS 100
#define NB_DECOMPRESSION_ALGORITHMS 100
@@ -446,7 +479,14 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
for (i=0; i<nbChunks; i++) {
chunkP[i].id = (U32)i;
chunkP[i].origBuffer = in; in += g_chunkSize;
- if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ assert(g_chunkSize > 0);
+ if (remaining > (size_t)g_chunkSize) {
+ chunkP[i].origSize = g_chunkSize;
+ remaining -= (size_t)g_chunkSize;
+ } else {
+ chunkP[i].origSize = (int)remaining;
+ remaining = 0;
+ }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
@@ -501,9 +541,10 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
if (initFunction!=NULL) initFunction();
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
- if (chunkP[chunkNb].compressedSize==0)
- DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1);
- }
+ if (chunkP[chunkNb].compressedSize==0) {
+ DISPLAY("ERROR ! %s() = 0 !! \n", compressorName);
+ exit(1);
+ } }
nb_loops++;
}
clockTime = BMK_GetClockSpan(clockTime);
@@ -511,7 +552,7 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
nb_loops += !nb_loops; /* avoid division by zero */
averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC;
if (averageTime < bestTime) bestTime = averageTime;
- cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
+ cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += (size_t)chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
}
@@ -531,23 +572,30 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
for (i=0; i<nbChunks; i++) {
- chunkP[i].id = i;
+ chunkP[i].id = (U32)i;
chunkP[i].origBuffer = in; in += g_chunkSize;
- if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ if ((int)remaining > g_chunkSize) {
+ chunkP[i].origSize = g_chunkSize;
+ remaining -= (size_t)g_chunkSize;
+ } else {
+ chunkP[i].origSize = (int)remaining;
+ remaining = 0;
+ }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
}
}
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize);
- if (chunkP[chunkNb].compressedSize==0)
- DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1);
- }
+ if (chunkP[chunkNb].compressedSize==0) {
+ DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress");
+ exit(1);
+ } }
/* Decompression Algorithms */
for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
- const char* dName;
- int (*decompressionFunction)(const char*, char*, int, int);
+ const char* dName = NULL;
+ int (*decompressionFunction)(const char*, char*, int, int) = NULL;
double bestTime = 100000000.;
int checkResult = 1;
@@ -565,17 +613,15 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
#ifndef LZ4_DLL_IMPORT
case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
#endif
- case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress";
- { size_t const errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL);
- if (LZ4F_isError(errorCode)) {
- DISPLAY("Error while preparing compressed frame\n");
- free(orig_buff);
- free(compressed_buff);
- free(chunkP);
- return 1;
- }
+ case 10:
+ case 11:
+ if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
+ if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
+ /* prepare compressed data using frame format */
+ { size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
+ assert(!LZ4F_isError(fcsize));
chunkP[0].origSize = (int)benchedSize;
- chunkP[0].compressedSize = (int)errorCode;
+ chunkP[0].compressedSize = (int)fcsize;
nbChunks = 1;
break;
}
@@ -583,6 +629,9 @@ int fullSpeedBench(const char** fileNamesTable, int nbFiles)
continue; /* skip if unknown ID */
}
+ assert(decompressionFunction != NULL);
+ assert(dName != NULL);
+
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a5b5c93..8a095c4 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -122,6 +122,14 @@ static U32 FUZ_rotl32(U32 u32, U32 nbBits)
return ((u32 << nbBits) | (u32 >> (32 - nbBits)));
}
+static U32 FUZ_highbit32(U32 v32)
+{
+ unsigned nbBits = 0;
+ if (v32==0) return 0;
+ while (v32) { v32 >>= 1; nbBits++; }
+ return nbBits;
+}
+
static U32 FUZ_rand(U32* src)
{
U32 rand32 = *src;
@@ -207,7 +215,7 @@ static int FUZ_AddressOverflow(void)
}
{ size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
- unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1);
+ int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
char* const input = buffers[nbBuff-1];
char* output = buffers[nbBuff];
int r;
@@ -215,7 +223,7 @@ static int FUZ_AddressOverflow(void)
input[1] = (char)0xFF;
input[2] = (char)0xFF;
input[3] = (char)0xFF;
- { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
+ { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
input[0] = (char)0x1F; /* Match length overflow */
@@ -366,7 +374,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
U32 testNb = 0;
U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
- int const blockStart = (FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
+ int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
int const dictSize = MIN(dictSizeRand, blockStart - 1);
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
@@ -388,69 +396,65 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test compression destSize */
FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
- { int srcSize = blockSize;
- int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
- char endCheck = FUZ_rand(&randState) & 255;
+ { int cSize, srcSize = blockSize;
+ int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
+ char const endCheck = (char)(FUZ_rand(&randState) & 255);
compressedBuffer[targetSize] = endCheck;
- ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
- FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
+ cSize = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
+ FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
- FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
- DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
+ FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() read more than src buffer !");
+ DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", cSize, targetSize, srcSize, blockSize);
if (targetSize>0) {
/* check correctness */
U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
- char const canary = FUZ_rand(&randState) & 255;
- FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
+ char const canary = (char)(FUZ_rand(&randState) & 255);
+ FUZ_CHECKTEST((cSize==0), "LZ4_compress_destSize() compression failed");
FUZ_DISPLAYTEST();
- compressedSize = ret;
decodedBuffer[srcSize] = canary;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
- FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+ { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
+ FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
+ FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+ }
FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
- { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
- FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
-
- DISPLAYLEVEL(5, " OK \n");
- } else {
- DISPLAYLEVEL(5, " \n");
- } }
+ { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
+ FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
+ } }
+ DISPLAYLEVEL(5, " OK \n");
+ }
/* Test compression HC destSize */
FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
- { int srcSize = blockSize;
- int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
- char const endCheck = FUZ_rand(&randState) & 255;
- void* ctx = LZ4_createHC(block);
+ { int cSize, srcSize = blockSize;
+ int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
+ char const endCheck = (char)(FUZ_rand(&randState) & 255);
+ void* const ctx = LZ4_createHC(block);
FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
compressedBuffer[targetSize] = endCheck;
- ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
+ cSize = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
- compressionLevel, ret, targetSize, srcSize, blockSize);
+ compressionLevel, cSize, targetSize, srcSize, blockSize);
LZ4_freeHC(ctx);
- FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
+ FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
if (targetSize>0) {
/* check correctness */
U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
- char const canary = FUZ_rand(&randState) & 255;
- FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
+ char const canary = (char)(FUZ_rand(&randState) & 255);
+ FUZ_CHECKTEST((cSize==0), "LZ4_compress_HC_destSize() compression failed");
FUZ_DISPLAYTEST();
- compressedSize = ret;
decodedBuffer[srcSize] = canary;
- ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
- FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+ { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
+ FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
+ FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
+ }
FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
- { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
+ { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
- }
- DISPLAYLEVEL(5, " OK \n");
- } else {
- DISPLAYLEVEL(5, " \n");
- } }
+ } }
+ DISPLAYLEVEL(5, " OK \n");
+ }
/* Test compression HC */
FUZ_DISPLAYTEST("test LZ4_compress_HC()");
@@ -524,7 +528,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data");
}
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
}
@@ -559,13 +563,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
}
+ /* noisy src decompression test */
+
+ /* insert noise into src */
+ { U32 const maxNbBits = FUZ_highbit32((U32)compressedSize);
+ size_t pos = 0;
+ for (;;) {
+ /* keep some original src */
+ { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
+ size_t const mask = (1<<nbBits) - 1;
+ size_t const skipLength = FUZ_rand(&randState) & mask;
+ pos += skipLength;
+ }
+ if (pos >= (size_t)compressedSize) break;
+ /* add noise */
+ { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
+ U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
+ size_t const mask = (1<<nbBits) - 1;
+ size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
+ size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos);
+ size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength);
+ memcpy(cBuffer_exact + pos, (const char*)CNBuffer + noiseStart, noiseLength);
+ pos += noiseLength;
+ } } }
+
+ /* decompress noisy source */
+ FUZ_DISPLAYTEST("decompress noisy source ");
+ { U32 const endMark = 0xA9B1C3D6;
+ memcpy(decodedBuffer+blockSize, &endMark, sizeof(endMark));
+ { int const decompressResult = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize);
+ /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
+ FUZ_CHECKTEST(decompressResult > blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize);
+ }
+ { U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck));
+ FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow");
+ } } /* noisy src decompression test */
+
free(cBuffer_exact);
}
/* Test decoding with input size being one byte too short => must fail */
FUZ_DISPLAYTEST();
{ int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
- FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
+ FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize);
}
/* Test decoding with input size being one byte too large => must fail */
@@ -578,7 +618,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test partial decoding => must work */
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
- { size_t const missingBytes = FUZ_rand(&randState) % blockSize;
+ { size_t const missingBytes = FUZ_rand(&randState) % (unsigned)blockSize;
int const targetSize = (int)((size_t)blockSize - missingBytes);
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, targetSize, blockSize);
@@ -615,8 +655,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
compressedBuffer[compressedSize-missingBytes] = 0;
- ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
- FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
+ { int const cSize = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
+ FUZ_CHECKTEST(cSize, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
+ }
FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes)
}
@@ -626,8 +667,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
compressedBuffer[HCcompressedSize-missingBytes] = 0;
- ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
- FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
+ { int const hcSize = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
+ FUZ_CHECKTEST(hcSize, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
+ }
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes)
}
@@ -654,8 +696,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (crcCheck!=crcOrig) {
FUZ_findDiff(block, decodedBuffer);
EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
- }
- }
+ } }
FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
@@ -672,7 +713,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
- FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary but with an output buffer too short by one byte");
+ FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte");
LZ4_loadDict(&LZ4dictBody, dict, dictSize);
ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
@@ -697,15 +738,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
if (crcCheck!=crcOrig) {
FUZ_findDiff(block, decodedBuffer);
EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
- }
- }
+ } }
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
}
@@ -722,17 +762,16 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST();
- { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
- if ((U32)blockSize > missingBytes) {
+ { int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
+ if (blockSize > missingBytes) {
decodedBuffer[blockSize-missingBytes] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
- FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes);
+ FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize);
} }
/* Compress using external dictionary stream */
- {
- LZ4_stream_t LZ4_stream;
+ { LZ4_stream_t LZ4_stream;
int expectedSize;
U32 expectedCrc;
@@ -740,7 +779,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
LZ4_loadDict(&LZ4dictBody, dict, dictSize);
expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
- expectedCrc = XXH32(compressedBuffer, expectedSize, 0);
+ expectedCrc = XXH32(compressedBuffer, (size_t)expectedSize, 0);
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
LZ4_loadDict(&LZ4dictBody, dict, dictSize);
@@ -756,7 +795,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
* test.
*/
FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
- FUZ_CHECKTEST(XXH32(compressedBuffer, blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+ FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
LZ4_resetStream_fast(&LZ4_stream);
@@ -772,7 +811,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
- FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+ FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
FUZ_DISPLAYTEST();
@@ -782,7 +821,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
- FUZ_CHECKTEST(XXH32(compressedBuffer, ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
+ FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
FUZ_CHECKTEST(LZ4_stream.internal_donotuse.dirty, "context should be good");
}
@@ -792,12 +831,11 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
- { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0);
+ { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
if (crcCheck!=crcOrig) {
FUZ_findDiff(block, decodedBuffer);
EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
- }
- }
+ } }
FUZ_DISPLAYTEST();
decodedBuffer[blockSize] = 0;
@@ -823,7 +861,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
{ U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
if ((U32)blockSize > missingBytes) {
- decodedBuffer[blockSize-missingBytes] = 0;
+ decodedBuffer[(U32)blockSize-missingBytes] = 0;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
@@ -965,8 +1003,8 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
}
-#define testInputSize (192 KB)
-#define testCompressedSize (128 KB)
+#define testInputSize (196 KB)
+#define testCompressedSize (130 KB)
#define ringBufferSize (8 KB)
static void FUZ_unitTests(int compressionLevel)
@@ -990,17 +1028,17 @@ static void FUZ_unitTests(int compressionLevel)
FUZ_AddressOverflow();
/* Test decoding with empty input */
- DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input");
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n");
LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize);
/* Test decoding with a one byte input */
- DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input");
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n");
{ char const tmp = (char)0xFF;
LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize);
}
/* Test decoding shortcut edge case */
- DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case");
+ DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n");
{ char tmp[17];
/* 14 bytes of literals, followed by a 14 byte match.
* Should not read beyond the end of the buffer.
@@ -1013,6 +1051,54 @@ static void FUZ_unitTests(int compressionLevel)
FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail");
} }
+ /* in-place compression test */
+ DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :");
+ { int const sampleSize = 65 KB;
+ int const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
+ int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
+ int const startInputIndex = outSize - sampleSize;
+ char* const startInput = testCompressed + startInputIndex;
+ XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
+ int cSize;
+ assert(outSize < (int)testCompressedSize);
+ memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */
+ /* compress in-place */
+ cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize);
+ assert(cSize != 0); /* ensure compression is successful */
+ assert(maxCSize < INT_MAX);
+ assert(cSize <= maxCSize);
+ /* decompress and verify */
+ { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
+ assert(dSize == sampleSize); /* correct size */
+ { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0);
+ assert(crcCheck == crcOrig);
+ } } }
+ DISPLAYLEVEL(3, " OK \n");
+
+ /* in-place decompression test */
+ DISPLAYLEVEL(3, "in-place decompression, limit case:");
+ { int const sampleSize = 65 KB;
+
+ FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState);
+ memset(testInput, 0, 267); /* calculated exactly so that compressedSize == originalSize-1 */
+
+ { XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0);
+ int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize);
+ assert(cSize == sampleSize - 1); /* worst case for in-place decompression */
+
+ { int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize);
+ int const startInputIndex = bufferSize - cSize;
+ char* const startInput = testVerify + startInputIndex;
+ memcpy(startInput, testCompressed, cSize);
+
+ /* decompress and verify */
+ { int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize);
+ assert(dSize == sampleSize); /* correct size */
+ { XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0);
+ assert(crcCheck == crcOrig);
+ } } } } }
+ DISPLAYLEVEL(3, " OK \n");
+
/* LZ4 streaming tests */
{ LZ4_stream_t streamingState;
U64 crcOrig;
@@ -1061,17 +1147,17 @@ static void FUZ_unitTests(int compressionLevel)
crcOrig = XXH64_digest(&xxhOrig);
memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
- compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
+ compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1);
FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed");
- result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+ result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewSafe);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
- result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+ result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize);
FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
@@ -1151,12 +1237,12 @@ static void FUZ_unitTests(int compressionLevel)
{ U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
- result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
- FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
- FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
-
- result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB);
- FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
+ { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
+ FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize);
+ FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
+ { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB);
+ FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
+ } }
{ U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
} }
@@ -1165,7 +1251,8 @@ static void FUZ_unitTests(int compressionLevel)
/* multiple HC compression test with dictionary */
{ int result1, result2;
int segSize = testCompressedSize / 2;
- U64 const crc64 = XXH64(testInput + segSize, testCompressedSize, 0);
+ XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) ,
+ XXH64(testInput + segSize, testCompressedSize, 0) );
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, segSize);
result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
@@ -1179,7 +1266,7 @@ static void FUZ_unitTests(int compressionLevel)
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
- { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
+ { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
} }
@@ -1201,15 +1288,15 @@ static void FUZ_unitTests(int compressionLevel)
{ XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
const char* dict = testInput + 3;
- int dictSize = (FUZ_rand(&randState) & 8191);
+ size_t dictSize = (FUZ_rand(&randState) & 8191);
char* dst = testVerify;
- size_t segStart = (size_t)dictSize + 7;
- int segSize = (FUZ_rand(&randState) & 8191);
+ size_t segStart = dictSize + 7;
+ size_t segSize = (FUZ_rand(&randState) & 8191);
int segNb = 1;
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
- LZ4_loadDictHC(&sHC, dict, dictSize);
+ LZ4_loadDictHC(&sHC, dict, (int)dictSize);
XXH64_reset(&crcOrigState, 0);
XXH64_reset(&crcNewState, 0);
@@ -1217,29 +1304,28 @@ static void FUZ_unitTests(int compressionLevel)
while (segStart + segSize < testInputSize) {
XXH64_update(&crcOrigState, testInput + segStart, segSize);
crcOrig = XXH64_digest(&crcOrigState);
- result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize));
+ assert(segSize <= INT_MAX);
+ result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize));
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
- result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize);
- FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
+ result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize);
+ FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb);
XXH64_update(&crcNewState, dst, segSize);
{ U64 const crcNew = XXH64_digest(&crcNewState);
if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
}
- assert(segSize >= 0);
dict = dst;
dictSize = segSize;
- dst += (size_t)segSize + 1;
+ dst += segSize + 1;
segNb ++;
- segStart += (size_t)segSize + (FUZ_rand(&randState) & 0xF) + 1;
+ segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1;
segSize = (FUZ_rand(&randState) & 8191);
- }
- }
+ } }
/* ring buffer test */
{ XXH64_state_t xxhOrig;
@@ -1266,18 +1352,21 @@ static void FUZ_unitTests(int compressionLevel)
crcOrig = XXH64_digest(&xxhOrig);
memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
- compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+ assert(messageSize < INT_MAX);
+ compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
- result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, messageSize);
+ assert(messageSize < INT_MAX);
+ result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewSafe);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
- result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, messageSize);
+ assert(messageSize < INT_MAX);
+ result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize);
FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
@@ -1337,14 +1426,14 @@ static void FUZ_unitTests(int compressionLevel)
result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize);
FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
- XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
+ XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewSafe);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
- XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
+ XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewFast);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
@@ -1360,7 +1449,7 @@ static void FUZ_unitTests(int compressionLevel)
dNext = 0;
while (totalMessageSize < 9 MB) {
- XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize);
crcOrig = XXH64_digest(&xxhOrig);
compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
@@ -1375,7 +1464,7 @@ static void FUZ_unitTests(int compressionLevel)
testCompressed, ringBufferSafe + dNext,
compressedSize, dBufferSize - dNext); /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */
FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
- XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, messageSize);
+ XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewSafe);
if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test");
@@ -1384,7 +1473,7 @@ static void FUZ_unitTests(int compressionLevel)
/* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */
result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed");
- XXH64_update(&xxhNewFast, ringBufferFast + dNext, messageSize);
+ XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNewFast);
if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test");
@@ -1392,7 +1481,8 @@ static void FUZ_unitTests(int compressionLevel)
/* prepare next message */
dNext += messageSize;
- totalMessageSize += messageSize;
+ assert(messageSize >= 0);
+ totalMessageSize += (unsigned)messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535);
if (dNext + maxMessageSize > dBufferSize) dNext = 0;
@@ -1410,6 +1500,11 @@ static void FUZ_unitTests(int compressionLevel)
}
+
+/* =======================================
+ * CLI
+ * ======================================= */
+
static int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
@@ -1433,8 +1528,8 @@ int main(int argc, const char** argv)
U32 seed = 0;
int seedset = 0;
int argNb;
- int nbTests = NB_ATTEMPTS;
- int testNb = 0;
+ unsigned nbTests = NB_ATTEMPTS;
+ unsigned testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int use_pause = 0;
const char* programName = argv[0];
@@ -1472,7 +1567,7 @@ int main(int argc, const char** argv)
nbTests = 0; duration = 0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
- nbTests += *argument - '0';
+ nbTests += (unsigned)(*argument - '0');
argument++;
}
break;
@@ -1495,7 +1590,7 @@ int main(int argc, const char** argv)
case '6':
case '7':
case '8':
- case '9': duration *= 10; duration += *argument++ - '0'; continue;
+ case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
}
break;
}
@@ -1506,7 +1601,7 @@ int main(int argc, const char** argv)
seed=0; seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
- seed += *argument - '0';
+ seed += (U32)(*argument - '0');
argument++;
}
break;
@@ -1516,7 +1611,7 @@ int main(int argc, const char** argv)
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
- testNb += *argument - '0';
+ testNb += (unsigned)(*argument - '0');
argument++;
}
break;
@@ -1551,7 +1646,7 @@ int main(int argc, const char** argv)
if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
- if (nbTests<=0) nbTests=1;
+ nbTests += (nbTests==0); /* avoid zero */
{ int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
if (use_pause) {
diff --git a/tests/test-lz4-list.py b/tests/test-lz4-list.py
new file mode 100644
index 0000000..ce89757
--- /dev/null
+++ b/tests/test-lz4-list.py
@@ -0,0 +1,282 @@
+#! /usr/bin/env python3
+import subprocess
+import time
+import glob
+import os
+import tempfile
+import unittest
+
+SIZES = [3, 11] # Always 2 sizes
+MIB = 1048576
+LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4"
+if not os.path.exists(LZ4):
+ LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4"
+TEMP = tempfile.gettempdir()
+
+
+class NVerboseFileInfo(object):
+ def __init__(self, line_in):
+ self.line = line_in
+ splitlines = line_in.split()
+ if len(splitlines) != 7:
+ errout("Unexpected line: {}".format(line_in))
+ self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines
+ self.exp_unc_size = 0
+ # Get real file sizes
+ if "concat-all" in self.filename or "2f--content-size" in self.filename:
+ for i in SIZES:
+ self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i))
+ else:
+ uncompressed_filename = self.filename.split("-")[0]
+ self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename))
+ self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename))
+
+
+class TestNonVerbose(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.nvinfo_list = []
+ for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)):
+ if i > 0:
+ self.nvinfo_list.append(NVerboseFileInfo(line))
+
+ def test_frames(self):
+ all_concat_frames = 0
+ all_concat_index = None
+ for i, nvinfo in enumerate(self.nvinfo_list):
+ if "concat-all" in nvinfo.filename:
+ all_concat_index = i
+ elif "2f--content-size" in nvinfo.filename:
+ self.assertEqual("2", nvinfo.frames, nvinfo.line)
+ all_concat_frames += 2
+ else:
+ self.assertEqual("1", nvinfo.frames, nvinfo.line)
+ all_concat_frames += 1
+ self.assertNotEqual(None, all_concat_index, "Couldn't find concat-all file index.")
+ self.assertEqual(self.nvinfo_list[all_concat_index].frames, str(all_concat_frames), self.nvinfo_list[all_concat_index].line)
+
+ def test_frame_types(self):
+ for nvinfo in self.nvinfo_list:
+ if "-lz4f-" in nvinfo.filename:
+ self.assertEqual(nvinfo.type, "LZ4Frame", nvinfo.line)
+ elif "-legc-" in nvinfo.filename:
+ self.assertEqual(nvinfo.type, "LegacyFrame", nvinfo.line)
+ elif "-skip-" in nvinfo.filename:
+ self.assertEqual(nvinfo.type, "SkippableFrame", nvinfo.line)
+
+ def test_block(self):
+ for nvinfo in self.nvinfo_list:
+ # if "-leg" in nvinfo.filename or "-skip" in nvinfo.filename:
+ # self.assertEqual(nvinfo.block, "-", nvinfo.line)
+ if "--BD" in nvinfo.filename:
+ self.assertRegex(nvinfo.block, "^B[0-9]+D$", nvinfo.line)
+ elif "--BI" in nvinfo.filename:
+ self.assertRegex(nvinfo.block, "^B[0-9]+I$", nvinfo.line)
+
+ def test_compressed_size(self):
+ for nvinfo in self.nvinfo_list:
+ self.assertEqual(nvinfo.compressed, to_human(nvinfo.exp_comp_size), nvinfo.line)
+
+ def test_ratio(self):
+ for nvinfo in self.nvinfo_list:
+ if "--content-size" in nvinfo.filename:
+ self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line)
+
+ def test_uncompressed_size(self):
+ for nvinfo in self.nvinfo_list:
+ if "--content-size" in nvinfo.filename:
+ self.assertEqual(nvinfo.uncompressed, to_human(nvinfo.exp_unc_size), nvinfo.line)
+
+
+class VerboseFileInfo(object):
+ def __init__(self, lines):
+ # Parse lines
+ self.frame_list = []
+ self.file_frame_map = []
+ for i, line in enumerate(lines):
+ if i == 0:
+ self.filename = line
+ continue
+ elif i == 1:
+ # Skip header
+ continue
+ frame_info = dict(zip(["frame", "type", "block", "checksum", "compressed", "uncompressed", "ratio"], line.split()))
+ frame_info["line"] = line
+ self.frame_list.append(frame_info)
+
+
+class TestVerbose(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ # Even do we're listing 2 files to test multiline working as expected.
+ # we're only really interested in testing the output of the concat-all file.
+ self.vinfo_list = []
+ start = end = 0
+ output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True)
+ for i, line in enumerate(output):
+ if line.startswith("test_list"):
+ if start != 0 and end != 0:
+ self.vinfo_list.append(VerboseFileInfo(output[start:end]))
+ start = i
+ if not line:
+ end = i
+ self.vinfo_list.append(VerboseFileInfo(output[start:end]))
+ # Populate file_frame_map as a reference of the expected info
+ concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4")
+ # One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
+ for i, filename in enumerate(concat_file_list):
+ if "2f--content-size" in filename:
+ concat_file_list.insert(i, filename)
+ break
+ self.cvinfo = self.vinfo_list[0]
+ self.cvinfo.file_frame_map = concat_file_list
+ self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP))
+
+ def test_filename(self):
+ for i, vinfo in enumerate(self.vinfo_list):
+ self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list)))
+
+ def test_frame_number(self):
+ for vinfo in self.vinfo_list:
+ for i, frame_info in enumerate(vinfo.frame_list):
+ self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"])
+
+ def test_frame_type(self):
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ if "-lz4f-" in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"])
+ elif "-legc-" in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"])
+ elif "-skip-" in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"])
+
+ def test_block(self):
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ if "--BD" in self.cvinfo.file_frame_map[i]:
+ self.assertRegex(self.cvinfo.frame_list[i]["block"], "^B[0-9]+D$", self.cvinfo.frame_list[i]["line"])
+ elif "--BI" in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]["block"], "^B[0-9]+I$", self.cvinfo.frame_list[i]["line"])
+
+ def test_checksum(self):
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"])
+
+ def test_compressed(self):
+ total = 0
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ if "-2f-" not in self.cvinfo.file_frame_map[i]:
+ expected_size = os.path.getsize(self.cvinfo.file_frame_map[i])
+ self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"])
+ total += int(self.cvinfo.frame_list[i]["compressed"])
+ self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename))
+
+ def test_uncompressed(self):
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ ffm = self.cvinfo.file_frame_map[i]
+ if "-2f-" not in ffm and "--content-size" in ffm:
+ expected_size_unc = int(ffm[ffm.rindex("_") + 1:ffm.index("M")]) * 1048576
+ self.assertEqual(self.cvinfo.frame_list[i]["uncompressed"], str(expected_size_unc), self.cvinfo.frame_list[i]["line"])
+
+ def test_ratio(self):
+ for i, frame_info in enumerate(self.cvinfo.frame_list):
+ if "--content-size" in self.cvinfo.file_frame_map[i]:
+ self.assertEqual(self.cvinfo.frame_list[i]['ratio'],
+ "{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100),
+ self.cvinfo.frame_list[i]["line"])
+
+
+def to_human(size):
+ for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']:
+ if size < 1024.0:
+ break
+ size /= 1024.0
+ return "{:.2f}{}".format(size, unit)
+
+
+def log(text):
+ print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
+
+
+def errout(text, err=1):
+ log(text)
+ exit(err)
+
+
+def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
+ if os.environ.get('QEMU_SYS'):
+ command = "{} {}".format(os.environ['QEMU_SYS'], command)
+ if print_command:
+ log("> " + command)
+ popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell)
+ stdout_lines, stderr_lines = popen.communicate()
+ stderr_lines = stderr_lines.decode("utf-8")
+ stdout_lines = stdout_lines.decode("utf-8")
+ if print_output:
+ if stdout_lines:
+ print(stdout_lines)
+ if stderr_lines:
+ print(stderr_lines)
+ if popen.returncode is not None and popen.returncode != 0:
+ if stderr_lines and not print_output and print_error:
+ print(stderr_lines)
+ errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines))
+ return (stdout_lines + stderr_lines).splitlines()
+
+
+def cleanup(silent=False):
+ for f in glob.glob("{}/test_list*".format(TEMP)):
+ if not silent:
+ log("Deleting {}".format(f))
+ os.unlink(f)
+
+
+def datagen(file_name, size):
+ non_sparse_size = size // 2
+ sparse_size = size - non_sparse_size
+ with open(file_name, "wb") as f:
+ f.seek(sparse_size)
+ f.write(os.urandom(non_sparse_size))
+
+
+def generate_files():
+ # file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~
+ # Generate LZ4Frames
+ for i in SIZES:
+ filename = "{}/test_list_{}M".format(TEMP, i)
+ log("Generating {}".format(filename))
+ datagen(filename, i * MIB)
+ for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]:
+ lz4file = "{}-lz4f-1f{}.lz4".format(filename, j)
+ execute("{} {} {} {}".format(LZ4, j, filename, lz4file))
+ # Generate skippable frames
+ lz4file = "{}-skip-1f.lz4".format(filename)
+ skipsize = i * 1024
+ skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False)
+ with open(lz4file, 'wb') as f:
+ f.write(skipbytes)
+ f.write(os.urandom(skipsize))
+ # Generate legacy frames
+ lz4file = "{}-legc-1f.lz4".format(filename)
+ execute("{} -l {} {}".format(LZ4, filename, lz4file))
+
+ # Concatenate --content-size files
+ file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP))
+ with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile:
+ for fname in file_list:
+ with open(fname, 'rb') as infile:
+ outfile.write(infile.read())
+
+ # Concatenate all files
+ file_list = glob.glob("{}/test_list_*.lz4".format(TEMP))
+ with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile:
+ for fname in file_list:
+ with open(fname, 'rb') as infile:
+ outfile.write(infile.read())
+
+
+if __name__ == '__main__':
+ cleanup()
+ generate_files()
+ unittest.main(verbosity=2, exit=False)
+ cleanup(silent=True)
generated by cgit v0.12 at 2024-07-20 00:00:43 (GMT)