From b9836b2a030a3756b1f8cf1341875354c84dd207 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 19 Mar 2018 14:01:57 -0400 Subject: Restore Framebench Tool This reverts commit 70f14823a46719e81e808d9ed9df90f478bcfd3f. --- tests/Makefile | 6 +- tests/framebench.c | 296 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 tests/framebench.c diff --git a/tests/Makefile b/tests/Makefile index 2b93c9f..f6a4ff3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen +all: fullbench fuzzer frametest datagen framebench all32: CFLAGS+=-m32 all32: all @@ -99,6 +99,9 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) @@ -110,6 +113,7 @@ clean: fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ + framebench$(EXT) \ fasttest$(EXT) datagen$(EXT) checkTag$(EXT) @rm -fR $(TESTDIR) @echo Cleaning completed diff --git a/tests/framebench.c b/tests/framebench.c new file mode 100644 index 0000000..18b5ff3 --- /dev/null +++ b/tests/framebench.c @@ -0,0 +1,296 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#include "lz4frame.h" +#include "lz4frame_static.h" + +#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } } + +typedef struct { + size_t iter; + LZ4_stream_t *ctx; + LZ4F_cctx *cctx; + char *obuf; + size_t osize; + const char* ibuf; + size_t isize; + size_t num_ibuf; + const LZ4F_CDict* cdict; + LZ4F_preferences_t* prefs; + const LZ4F_compressOptions_t* options; +} bench_params_t; + +size_t compress_frame(bench_params_t *p) { + size_t iter = p->iter; + LZ4F_cctx *cctx = p->cctx; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + const LZ4F_CDict* cdict = p->cdict; + LZ4F_preferences_t* prefs = p->prefs; + + size_t oused; + + prefs->frameInfo.contentSize = isize; + + oused = LZ4F_compressFrame_usingCDict( + cctx, + obuf, + osize, + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, + isize, + cdict, + prefs); + LZ4F_CHECK(oused); + + return oused; +} + +size_t compress_begin(bench_params_t *p) { + size_t iter = p->iter; + LZ4F_cctx *cctx = p->cctx; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + const LZ4F_CDict* cdict = p->cdict; + LZ4F_preferences_t* prefs = p->prefs; + const LZ4F_compressOptions_t* options = p->options; + + char *oend = obuf + osize; + size_t oused; + + prefs->frameInfo.contentSize = isize; + + oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs); + LZ4F_CHECK(oused); + obuf += oused; + oused = LZ4F_compressUpdate( + cctx, + obuf, + oend - obuf, + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, + isize, + options); + LZ4F_CHECK(oused); + obuf += oused; + oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options); + LZ4F_CHECK(oused); + + return obuf - p->obuf; +} + +size_t compress_default(bench_params_t *p) { + size_t iter = p->iter; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + + char *oend = obuf + osize; + size_t oused; + + oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf); + obuf += oused; + + return obuf - p->obuf; +} + +size_t compress_extState(bench_params_t *p) { + size_t iter = p->iter; + LZ4_stream_t *ctx = p->ctx; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + + char *oend = obuf + osize; + size_t oused; + + oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0); + obuf += oused; + + return obuf - p->obuf; +} + +uint64_t bench( + size_t (*fun)(bench_params_t *), + uint64_t repetitions, + bench_params_t *params, + size_t *osizePtr +) { + struct timespec start, end; + size_t i, osize = 0; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0; + + for (i = 0; i < repetitions; i++) { + size_t o; + params->iter = i; + o = fun(params); + if (!o) return 0; + osize += o; + } + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0; + + *osizePtr = osize / repetitions; + return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); +} + +int main(int argc, char *argv[]) { + + + struct stat st; + size_t bytes_read; + + const char *dict_fn; + size_t dict_size; + char *dict_buf; + FILE *dict_file; + + const char *in_fn; + size_t in_size; + size_t num_in_buf; + size_t cur_in_buf; + char *in_buf; + FILE *in_file; + + size_t out_size; + char *out_buf; + + LZ4_stream_t *ctx; + LZ4F_cctx *cctx; + LZ4F_CDict *cdict; + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; + + size_t out_used; + + uint64_t time_taken; + uint64_t repetitions; + + bench_params_t params; + + if (argc != 3) return 1; + dict_fn = argv[1]; + in_fn = argv[2]; + + if (stat(dict_fn, &st)) return 1; + dict_size = st.st_size; + dict_buf = (char *)malloc(dict_size); + if (!dict_buf) return 1; + dict_file = fopen(dict_fn, "r"); + bytes_read = fread(dict_buf, 1, dict_size, dict_file); + if (bytes_read != dict_size) return 1; + + if (stat(in_fn, &st)) return 1; + in_size = st.st_size; + num_in_buf = 256 * 1024 * 1024 / in_size; + if (num_in_buf == 0) { + num_in_buf = 1; + } + + in_buf = (char *)malloc(in_size * num_in_buf); + if (!in_buf) return 1; + in_file = fopen(in_fn, "r"); + bytes_read = fread(in_buf, 1, in_size, in_file); + if (bytes_read != in_size) return 1; + + for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) { + memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size); + } + + if (in_size <= 1024) { + repetitions = 100000; + } else + if (in_size <= 16384) { + repetitions = 10000; + } else + if (in_size <= 131072) { + repetitions = 1000; + } else + if (in_size <= 1048576) { + repetitions = 100; + } else { + repetitions = 50; + } + + memset(&prefs, 0, sizeof(prefs)); + prefs.autoFlush = 1; + if (in_size < 64 * 1024) + prefs.frameInfo.blockMode = LZ4F_blockIndependent; + prefs.frameInfo.contentSize = in_size; + + memset(&options, 0, sizeof(options)); + options.stableSrc = 1; + + out_size = LZ4F_compressFrameBound(in_size, &prefs); + out_buf = (char *)malloc(out_size); + if (!out_buf) return 1; + + if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1; + if (cctx == NULL) return 1; + + ctx = LZ4_createStream(); + if (ctx == NULL) return 1; + + cdict = LZ4F_createCDict(dict_buf, dict_size); + if (!cdict) return 1; + + fprintf(stderr, "dict size: %zd\n", dict_size); + fprintf(stderr, "input size: %zd\n", in_size); + + params.ctx = ctx; + params.cctx = cctx; + params.obuf = out_buf; + params.osize = out_size; + params.ibuf = in_buf; + params.isize = in_size; + params.num_ibuf = num_in_buf; + params.cdict = NULL; + params.prefs = &prefs; + params.options = &options; + + time_taken = bench(compress_default, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4_compress_default : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_extState, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4_compress_fast_extState : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4F_compressFrame : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4F_compressBegin : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + params.cdict = cdict; + + time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + + fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + return 0; +} \ No newline at end of file -- cgit v0.12 From 0bc13ab69c603346ff64050f7e3d3a1b3a5765a4 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 19 Mar 2018 15:00:59 -0400 Subject: Add HC Calls to Framebench --- tests/framebench.c | 99 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 17 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 18b5ff3..e2bcf40 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -10,6 +10,7 @@ #define LZ4_STATIC_LINKING_ONLY #include "lz4.h" +#include "lz4hc.h" #include "lz4frame.h" #include "lz4frame_static.h" @@ -18,12 +19,14 @@ typedef struct { size_t iter; LZ4_stream_t *ctx; + LZ4_streamHC_t *hcctx; LZ4F_cctx *cctx; char *obuf; size_t osize; const char* ibuf; size_t isize; size_t num_ibuf; + int clevel; const LZ4F_CDict* cdict; LZ4F_preferences_t* prefs; const LZ4F_compressOptions_t* options; @@ -117,11 +120,49 @@ size_t compress_extState(bench_params_t *p) { const char* ibuf = p->ibuf; size_t isize = p->isize; size_t num_ibuf = p->num_ibuf; + int clevel = p->clevel; char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, 0); + oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); + obuf += oused; + + return obuf - p->obuf; +} + +size_t compress_hc(bench_params_t *p) { + size_t iter = p->iter; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + int clevel = p->clevel; + + char *oend = obuf + osize; + size_t oused; + + oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); + obuf += oused; + + return obuf - p->obuf; +} + +size_t compress_hc_extState(bench_params_t *p) { + size_t iter = p->iter; + LZ4_streamHC_t *hcctx = p->hcctx; + char *obuf = p->obuf; + size_t osize = p->osize; + const char* ibuf = p->ibuf; + size_t isize = p->isize; + size_t num_ibuf = p->num_ibuf; + int clevel = p->clevel; + + char *oend = obuf + osize; + size_t oused; + + oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; @@ -174,6 +215,7 @@ int main(int argc, char *argv[]) { char *out_buf; LZ4_stream_t *ctx; + LZ4_streamHC_t *hcctx; LZ4F_cctx *cctx; LZ4F_CDict *cdict; LZ4F_preferences_t prefs; @@ -216,18 +258,18 @@ int main(int argc, char *argv[]) { } if (in_size <= 1024) { - repetitions = 100000; + repetitions = 10000; } else if (in_size <= 16384) { - repetitions = 10000; + repetitions = 1000; } else if (in_size <= 131072) { - repetitions = 1000; + repetitions = 100; } else if (in_size <= 1048576) { - repetitions = 100; + repetitions = 10; } else { - repetitions = 50; + repetitions = 5; } memset(&prefs, 0, sizeof(prefs)); @@ -249,6 +291,9 @@ int main(int argc, char *argv[]) { ctx = LZ4_createStream(); if (ctx == NULL) return 1; + hcctx = LZ4_createStreamHC(); + if (hcctx == NULL) return 1; + cdict = LZ4F_createCDict(dict_buf, dict_size); if (!cdict) return 1; @@ -256,41 +301,61 @@ int main(int argc, char *argv[]) { fprintf(stderr, "input size: %zd\n", in_size); params.ctx = ctx; + params.hcctx = hcctx; params.cctx = cctx; params.obuf = out_buf; params.osize = out_size; params.ibuf = in_buf; params.isize = in_size; params.num_ibuf = num_in_buf; + params.clevel = 1; params.cdict = NULL; params.prefs = &prefs; params.options = &options; time_taken = bench(compress_default, repetitions, ¶ms, &out_used); - - fprintf(stderr, "LZ4_compress_default : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + fprintf(stderr, "LZ4_compress_default : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); time_taken = bench(compress_extState, repetitions, ¶ms, &out_used); - - fprintf(stderr, "LZ4_compress_fast_extState : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + fprintf(stderr, "LZ4_compress_fast_extState : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); - - fprintf(stderr, "LZ4F_compressFrame : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + fprintf(stderr, "LZ4F_compressFrame : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); - - fprintf(stderr, "LZ4F_compressBegin : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + fprintf(stderr, "LZ4F_compressBegin : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); params.cdict = cdict; time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressFrame_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - fprintf(stderr, "LZ4F_compressFrame_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressBegin_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + params.cdict = NULL; + params.clevel = LZ4HC_CLEVEL_MIN; + params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN; + + time_taken = bench(compress_hc, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4_compress_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_hc_extState, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4_compress_HC_extStateHC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressFrame_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressBegin_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - fprintf(stderr, "LZ4F_compressBegin_usingCDict: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + params.cdict = cdict; + + time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + + time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); return 0; -} \ No newline at end of file +} -- cgit v0.12 From ffb2d8b0ed9615694c156ba1a3f51e9e789d3c4e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 22 Mar 2018 16:26:44 -0400 Subject: Check Compressed Buffer is Correct in Frame Bench --- tests/framebench.c | 79 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index e2bcf40..a802ce0 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -21,11 +21,16 @@ typedef struct { LZ4_stream_t *ctx; LZ4_streamHC_t *hcctx; LZ4F_cctx *cctx; + LZ4F_dctx *dctx; + const char *dictbuf; + size_t dictsize; char *obuf; size_t osize; const char* ibuf; size_t isize; size_t num_ibuf; + char *checkbuf; + size_t checksize; int clevel; const LZ4F_CDict* cdict; LZ4F_preferences_t* prefs; @@ -168,19 +173,46 @@ size_t compress_hc_extState(bench_params_t *p) { return obuf - p->obuf; } +size_t check_lz4(bench_params_t *p, size_t csize) { + (void)csize; + memset(p->checkbuf, 0xFF, p->checksize); + return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize) + && !memcmp(p->ibuf, p->checkbuf, p->isize); +} + +size_t check_lz4f(bench_params_t *p, size_t csize) { + size_t cp = 0; + size_t dp = 0; + size_t dsize = p->checksize; + size_t cleft = csize; + size_t dleft = dsize; + size_t ret; + memset(p->checkbuf, 0xFF, p->checksize); + LZ4F_resetDecompressionContext(p->dctx); + do { + ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL); + cp += cleft; + dp += dleft; + cleft = csize - cp; + dleft = dsize - dp; + if (LZ4F_isError(ret)) return 0; + } while (cleft); + return !memcmp(p->ibuf, p->checkbuf, p->isize); +} + uint64_t bench( size_t (*fun)(bench_params_t *), + size_t (*checkfun)(bench_params_t *, size_t), uint64_t repetitions, bench_params_t *params, size_t *osizePtr ) { struct timespec start, end; - size_t i, osize = 0; + size_t i, osize = 0, o = 0; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0; for (i = 0; i < repetitions; i++) { - size_t o; params->iter = i; o = fun(params); if (!o) return 0; @@ -189,6 +221,9 @@ uint64_t bench( if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0; + o = checkfun(params, o); + if (!o) return 0; + *osizePtr = osize / repetitions; return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); } @@ -214,9 +249,13 @@ int main(int argc, char *argv[]) { size_t out_size; char *out_buf; + size_t check_size; + char *check_buf; + LZ4_stream_t *ctx; LZ4_streamHC_t *hcctx; LZ4F_cctx *cctx; + LZ4F_dctx *dctx; LZ4F_CDict *cdict; LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; @@ -257,6 +296,10 @@ int main(int argc, char *argv[]) { memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size); } + check_size = in_size; + check_buf = (char *)malloc(check_size); + if (!check_buf) return 1; + if (in_size <= 1024) { repetitions = 10000; } else @@ -288,6 +331,9 @@ int main(int argc, char *argv[]) { if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1; if (cctx == NULL) return 1; + if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1; + if (cctx == NULL) return 1; + ctx = LZ4_createStream(); if (ctx == NULL) return 1; @@ -303,58 +349,63 @@ int main(int argc, char *argv[]) { params.ctx = ctx; params.hcctx = hcctx; params.cctx = cctx; + params.dctx = dctx; + params.dictbuf = dict_buf; + params.dictsize = dict_size; params.obuf = out_buf; params.osize = out_size; params.ibuf = in_buf; params.isize = in_size; params.num_ibuf = num_in_buf; + params.checkbuf = check_buf; + params.checksize = check_size; params.clevel = 1; params.cdict = NULL; params.prefs = &prefs; params.options = &options; - time_taken = bench(compress_default, repetitions, ¶ms, &out_used); + time_taken = bench(compress_default, check_lz4, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4_compress_default : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_extState, repetitions, ¶ms, &out_used); + time_taken = bench(compress_extState, check_lz4, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4_compress_fast_extState : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressFrame : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressBegin : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); params.cdict = cdict; - time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressFrame_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressBegin_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); params.cdict = NULL; params.clevel = LZ4HC_CLEVEL_MIN; params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN; - time_taken = bench(compress_hc, repetitions, ¶ms, &out_used); + time_taken = bench(compress_hc, check_lz4, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4_compress_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_hc_extState, repetitions, ¶ms, &out_used); + time_taken = bench(compress_hc_extState, check_lz4, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4_compress_HC_extStateHC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressFrame_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressBegin_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); params.cdict = cdict; - time_taken = bench(compress_frame, repetitions, ¶ms, &out_used); + time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - time_taken = bench(compress_begin, repetitions, ¶ms, &out_used); + time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); return 0; -- cgit v0.12 From d4ee7554fc79fecd5db83913536f5c856765bbed Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 28 Mar 2018 16:19:30 -0400 Subject: Print More Detailed Results Inside bench(), Add Compression Levels --- tests/framebench.c | 103 ++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index a802ce0..6d99b1e 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -111,7 +111,9 @@ size_t compress_default(bench_params_t *p) { char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_default(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf); + oused = LZ4_compress_default( + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, + isize, oend - obuf); obuf += oused; return obuf - p->obuf; @@ -130,7 +132,10 @@ size_t compress_extState(bench_params_t *p) { char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_fast_extState_fastReset(ctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); + oused = LZ4_compress_fast_extState_fastReset( + ctx, + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, + isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; @@ -148,7 +153,9 @@ size_t compress_hc(bench_params_t *p) { char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_HC(ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); + oused = LZ4_compress_HC( + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, + isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; @@ -167,7 +174,10 @@ size_t compress_hc_extState(bench_params_t *p) { char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_HC_extStateHC(hcctx, ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, isize, oend - obuf, clevel); + oused = LZ4_compress_HC_extStateHC( + hcctx, + ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, + isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; @@ -176,7 +186,8 @@ size_t compress_hc_extState(bench_params_t *p) { size_t check_lz4(bench_params_t *p, size_t csize) { (void)csize; memset(p->checkbuf, 0xFF, p->checksize); - return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize) + return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, + p->dictbuf, p->dictsize) && !memcmp(p->ibuf, p->checkbuf, p->isize); } @@ -190,7 +201,9 @@ size_t check_lz4f(bench_params_t *p, size_t csize) { memset(p->checkbuf, 0xFF, p->checksize); LZ4F_resetDecompressionContext(p->dctx); do { - ret = LZ4F_decompress_usingDict(p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, p->dictbuf, p->dictsize, NULL); + ret = LZ4F_decompress_usingDict( + p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, + p->dictbuf, p->dictsize, NULL); cp += cleft; dp += dleft; cleft = csize - cp; @@ -200,15 +213,17 @@ size_t check_lz4f(bench_params_t *p, size_t csize) { return !memcmp(p->ibuf, p->checkbuf, p->isize); } + uint64_t bench( + char *bench_name, size_t (*fun)(bench_params_t *), size_t (*checkfun)(bench_params_t *, size_t), uint64_t repetitions, - bench_params_t *params, - size_t *osizePtr + bench_params_t *params ) { struct timespec start, end; size_t i, osize = 0, o = 0; + size_t time_taken; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0; @@ -224,8 +239,19 @@ uint64_t bench( o = checkfun(params, o); if (!o) return 0; - *osizePtr = osize / repetitions; - return (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); + time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - + (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); + + fprintf( + stderr, + "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n", + bench_name, params->clevel, + params->isize, osize / repetitions, + repetitions, time_taken, time_taken / repetitions, + ((double) 1000 * params->isize * repetitions) / time_taken + ); + + return time_taken; } int main(int argc, char *argv[]) { @@ -260,9 +286,8 @@ int main(int argc, char *argv[]) { LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; - size_t out_used; + int clevels[] = {1, 2, 3, 6, 9, 10, 12}; - uint64_t time_taken; uint64_t repetitions; bench_params_t params; @@ -364,49 +389,23 @@ int main(int argc, char *argv[]) { params.prefs = &prefs; params.options = &options; - time_taken = bench(compress_default, check_lz4, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4_compress_default : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_extState, check_lz4, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4_compress_fast_extState : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressFrame : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressBegin : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - params.cdict = cdict; - - time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressFrame_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) { + params.clevel = clevels[clevelidx]; + params.prefs->compressionLevel = clevels[clevelidx]; + params.cdict = NULL; - time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressBegin_usingCDict : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + bench("LZ4_compress_default" , compress_default , check_lz4 , repetitions, ¶ms); + bench("LZ4_compress_fast_extState" , compress_extState , check_lz4 , repetitions, ¶ms); + bench("LZ4_compress_HC" , compress_hc , check_lz4 , repetitions, ¶ms); + bench("LZ4_compress_HC_extStateHC" , compress_hc_extState, check_lz4 , repetitions, ¶ms); + bench("LZ4F_compressFrame" , compress_frame , check_lz4f, repetitions, ¶ms); + bench("LZ4F_compressBegin" , compress_begin , check_lz4f, repetitions, ¶ms); - params.cdict = NULL; - params.clevel = LZ4HC_CLEVEL_MIN; - params.prefs->compressionLevel = LZ4HC_CLEVEL_MIN; - - time_taken = bench(compress_hc, check_lz4, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4_compress_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_hc_extState, check_lz4, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4_compress_HC_extStateHC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressFrame_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + params.cdict = cdict; - time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressBegin_HC : %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - params.cdict = cdict; - - time_taken = bench(compress_frame, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressFrame_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); - - time_taken = bench(compress_begin, check_lz4f, repetitions, ¶ms, &out_used); - fprintf(stderr, "LZ4F_compressBegin_usingCDict_HC: %8ld B -> %8ld B, %9ld ns/iter, %6.1lf MB/s\n", in_size, out_used, time_taken / repetitions, ((double) 1000 * in_size * repetitions) / time_taken); + bench("LZ4F_compressFrame_usingCDict", compress_frame , check_lz4f, repetitions, ¶ms); + bench("LZ4F_compressBegin_usingCDict", compress_begin , check_lz4f, repetitions, ¶ms); + } return 0; } -- cgit v0.12 From a9a62321ffd077d1c4694b2762e84b60b2396c2d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 28 Mar 2018 16:36:35 -0400 Subject: Auto-Calculate Appropriate Repetition Count --- tests/framebench.c | 67 +++++++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 6d99b1e..d5d2268 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -218,33 +218,41 @@ uint64_t bench( char *bench_name, size_t (*fun)(bench_params_t *), size_t (*checkfun)(bench_params_t *, size_t), - uint64_t repetitions, bench_params_t *params ) { struct timespec start, end; size_t i, osize = 0, o = 0; - size_t time_taken; + size_t time_taken = 0; + uint64_t total_repetitions = 0; + uint64_t repetitions = 2; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0; - for (i = 0; i < repetitions; i++) { - params->iter = i; - o = fun(params); - if (!o) return 0; - osize += o; - } + while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms + if (total_repetitions) { + repetitions = total_repetitions; // double previous + } + + for (i = 0; i < repetitions; i++) { + params->iter = i; + o = fun(params); + if (!o) return 0; + osize += o; + } - if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0; + if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0; + + time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - + (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); + total_repetitions += repetitions; + } o = checkfun(params, o); if (!o) return 0; - time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); - fprintf( stderr, - "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %12ld ns, %9ld ns/iter, %7.2lf MB/s\n", + "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n", bench_name, params->clevel, params->isize, osize / repetitions, repetitions, time_taken, time_taken / repetitions, @@ -288,8 +296,6 @@ int main(int argc, char *argv[]) { int clevels[] = {1, 2, 3, 6, 9, 10, 12}; - uint64_t repetitions; - bench_params_t params; if (argc != 3) return 1; @@ -325,21 +331,6 @@ int main(int argc, char *argv[]) { check_buf = (char *)malloc(check_size); if (!check_buf) return 1; - if (in_size <= 1024) { - repetitions = 10000; - } else - if (in_size <= 16384) { - repetitions = 1000; - } else - if (in_size <= 131072) { - repetitions = 100; - } else - if (in_size <= 1048576) { - repetitions = 10; - } else { - repetitions = 5; - } - memset(&prefs, 0, sizeof(prefs)); prefs.autoFlush = 1; if (in_size < 64 * 1024) @@ -394,17 +385,17 @@ int main(int argc, char *argv[]) { params.prefs->compressionLevel = clevels[clevelidx]; params.cdict = NULL; - bench("LZ4_compress_default" , compress_default , check_lz4 , repetitions, ¶ms); - bench("LZ4_compress_fast_extState" , compress_extState , check_lz4 , repetitions, ¶ms); - bench("LZ4_compress_HC" , compress_hc , check_lz4 , repetitions, ¶ms); - bench("LZ4_compress_HC_extStateHC" , compress_hc_extState, check_lz4 , repetitions, ¶ms); - bench("LZ4F_compressFrame" , compress_frame , check_lz4f, repetitions, ¶ms); - bench("LZ4F_compressBegin" , compress_begin , check_lz4f, repetitions, ¶ms); + bench("LZ4_compress_default" , compress_default , check_lz4 , ¶ms); + bench("LZ4_compress_fast_extState" , compress_extState , check_lz4 , ¶ms); + bench("LZ4_compress_HC" , compress_hc , check_lz4 , ¶ms); + bench("LZ4_compress_HC_extStateHC" , compress_hc_extState, check_lz4 , ¶ms); + bench("LZ4F_compressFrame" , compress_frame , check_lz4f, ¶ms); + bench("LZ4F_compressBegin" , compress_begin , check_lz4f, ¶ms); params.cdict = cdict; - bench("LZ4F_compressFrame_usingCDict", compress_frame , check_lz4f, repetitions, ¶ms); - bench("LZ4F_compressBegin_usingCDict", compress_begin , check_lz4f, repetitions, ¶ms); + bench("LZ4F_compressFrame_usingCDict", compress_frame , check_lz4f, ¶ms); + bench("LZ4F_compressBegin_usingCDict", compress_begin , check_lz4f, ¶ms); } return 0; -- cgit v0.12 From 09df7a05f9eb4471a5af7d6ca1bb37bea36ccf91 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 28 Mar 2018 17:16:23 -0400 Subject: Add Run Name to Frame Bench Output --- tests/framebench.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index d5d2268..185d186 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -17,6 +17,7 @@ #define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } } typedef struct { + const char *run_name; size_t iter; LZ4_stream_t *ctx; LZ4_streamHC_t *hcctx; @@ -252,8 +253,8 @@ uint64_t bench( fprintf( stderr, - "%-30s @ lvl %2d: %8ld B -> %8ld B, %8ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n", - bench_name, params->clevel, + "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n", + params->run_name, bench_name, params->clevel, params->isize, osize / repetitions, repetitions, time_taken, time_taken / repetitions, ((double) 1000 * params->isize * repetitions) / time_taken @@ -263,7 +264,7 @@ uint64_t bench( } int main(int argc, char *argv[]) { - + char *run_name; struct stat st; size_t bytes_read; @@ -298,9 +299,10 @@ int main(int argc, char *argv[]) { bench_params_t params; - if (argc != 3) return 1; - dict_fn = argv[1]; - in_fn = argv[2]; + if (argc != 4) return 1; + run_name = argv[1]; + dict_fn = argv[2]; + in_fn = argv[3]; if (stat(dict_fn, &st)) return 1; dict_size = st.st_size; @@ -362,6 +364,7 @@ int main(int argc, char *argv[]) { fprintf(stderr, "dict size: %zd\n", dict_size); fprintf(stderr, "input size: %zd\n", in_size); + params.run_name = run_name; params.ctx = ctx; params.hcctx = hcctx; params.cctx = cctx; -- cgit v0.12 From 66f0c29aa4d81586a4a621a99c763ab17028c270 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 4 Apr 2018 13:12:28 -0400 Subject: Fix Framebench Statistics --- tests/framebench.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 185d186..1060e99 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -255,9 +255,9 @@ uint64_t bench( stderr, "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n", params->run_name, bench_name, params->clevel, - params->isize, osize / repetitions, - repetitions, time_taken, time_taken / repetitions, - ((double) 1000 * params->isize * repetitions) / time_taken + params->isize, osize / total_repetitions, + total_repetitions, time_taken, time_taken / total_repetitions, + ((double) 1000 * params->isize * total_repetitions) / time_taken ); return time_taken; -- cgit v0.12 From 9d971fd5c3c1e18b225307ef88e529c9b3850fcb Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 4 Apr 2018 14:06:23 -0400 Subject: Switch to Unaligned Samples to Compress Different Blobs Each Time --- tests/framebench.c | 53 +++++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 1060e99..01e30b3 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -28,6 +28,7 @@ typedef struct { char *obuf; size_t osize; const char* ibuf; + const char* isample; size_t isize; size_t num_ibuf; char *checkbuf; @@ -39,13 +40,11 @@ typedef struct { } bench_params_t; size_t compress_frame(bench_params_t *p) { - size_t iter = p->iter; LZ4F_cctx *cctx = p->cctx; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; const LZ4F_CDict* cdict = p->cdict; LZ4F_preferences_t* prefs = p->prefs; @@ -57,7 +56,7 @@ size_t compress_frame(bench_params_t *p) { cctx, obuf, osize, - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, + isample, isize, cdict, prefs); @@ -67,13 +66,11 @@ size_t compress_frame(bench_params_t *p) { } size_t compress_begin(bench_params_t *p) { - size_t iter = p->iter; LZ4F_cctx *cctx = p->cctx; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; const LZ4F_CDict* cdict = p->cdict; LZ4F_preferences_t* prefs = p->prefs; const LZ4F_compressOptions_t* options = p->options; @@ -90,7 +87,7 @@ size_t compress_begin(bench_params_t *p) { cctx, obuf, oend - obuf, - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, + isample, isize, options); LZ4F_CHECK(oused); @@ -102,74 +99,61 @@ size_t compress_begin(bench_params_t *p) { } size_t compress_default(bench_params_t *p) { - size_t iter = p->iter; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; char *oend = obuf + osize; size_t oused; - oused = LZ4_compress_default( - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, - isize, oend - obuf); + oused = LZ4_compress_default(isample, obuf, isize, oend - obuf); obuf += oused; return obuf - p->obuf; } size_t compress_extState(bench_params_t *p) { - size_t iter = p->iter; LZ4_stream_t *ctx = p->ctx; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; int clevel = p->clevel; char *oend = obuf + osize; size_t oused; oused = LZ4_compress_fast_extState_fastReset( - ctx, - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, - isize, oend - obuf, clevel); + ctx, isample, obuf, isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; } size_t compress_hc(bench_params_t *p) { - size_t iter = p->iter; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; int clevel = p->clevel; char *oend = obuf + osize; size_t oused; oused = LZ4_compress_HC( - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, - isize, oend - obuf, clevel); + isample, obuf, isize, oend - obuf, clevel); obuf += oused; return obuf - p->obuf; } size_t compress_hc_extState(bench_params_t *p) { - size_t iter = p->iter; LZ4_streamHC_t *hcctx = p->hcctx; char *obuf = p->obuf; size_t osize = p->osize; - const char* ibuf = p->ibuf; + const char* isample = p->isample; size_t isize = p->isize; - size_t num_ibuf = p->num_ibuf; int clevel = p->clevel; char *oend = obuf + osize; @@ -177,7 +161,7 @@ size_t compress_hc_extState(bench_params_t *p) { oused = LZ4_compress_HC_extStateHC( hcctx, - ibuf + ((iter * 2654435761U) % num_ibuf) * isize, obuf, + isample, obuf, isize, oend - obuf, clevel); obuf += oused; @@ -189,7 +173,7 @@ size_t check_lz4(bench_params_t *p, size_t csize) { memset(p->checkbuf, 0xFF, p->checksize); return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, p->dictbuf, p->dictsize) - && !memcmp(p->ibuf, p->checkbuf, p->isize); + && !memcmp(p->isample, p->checkbuf, p->isize); } size_t check_lz4f(bench_params_t *p, size_t csize) { @@ -211,7 +195,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) { dleft = dsize - dp; if (LZ4F_isError(ret)) return 0; } while (cleft); - return !memcmp(p->ibuf, p->checkbuf, p->isize); + return !memcmp(p->isample, p->checkbuf, p->isize); } @@ -236,6 +220,11 @@ uint64_t bench( for (i = 0; i < repetitions; i++) { params->iter = i; + if (params->num_ibuf == 1) { + params->isample = params->ibuf; + } else { + params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize)); + } o = fun(params); if (!o) return 0; osize += o; @@ -295,7 +284,7 @@ int main(int argc, char *argv[]) { LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; - int clevels[] = {1, 2, 3, 6, 9, 10, 12}; + int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; bench_params_t params; -- cgit v0.12 From f646c512e99032cf4545be1ffcfd2a17323fb586 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 5 Apr 2018 18:08:13 -0400 Subject: Print Failure Message in Framebench --- tests/framebench.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 01e30b3..048df34 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -226,7 +226,14 @@ uint64_t bench( params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize)); } o = fun(params); - if (!o) return 0; + if (!o) { + fprintf( + stderr, + "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n", + params->run_name, bench_name, params->clevel, + params->isize); + return 0; + } osize += o; } @@ -238,7 +245,14 @@ uint64_t bench( } o = checkfun(params, o); - if (!o) return 0; + if (!o) { + fprintf( + stderr, + "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n", + params->run_name, bench_name, params->clevel, + params->isize); + return 0; + } fprintf( stderr, -- cgit v0.12 From 9dae661b1fcc665a78a9de5341e651310d489f5c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Apr 2018 12:39:31 -0400 Subject: Fix Cast --- tests/framebench.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/framebench.c b/tests/framebench.c index 048df34..70cfec5 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -200,7 +200,7 @@ size_t check_lz4f(bench_params_t *p, size_t csize) { uint64_t bench( - char *bench_name, + const char *bench_name, size_t (*fun)(bench_params_t *), size_t (*checkfun)(bench_params_t *, size_t), bench_params_t *params @@ -299,6 +299,7 @@ int main(int argc, char *argv[]) { LZ4F_compressOptions_t options; int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; + unsigned int clevelidx; bench_params_t params; @@ -386,7 +387,7 @@ int main(int argc, char *argv[]) { params.prefs = &prefs; params.options = &options; - for (unsigned int clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) { + for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) { params.clevel = clevels[clevelidx]; params.prefs->compressionLevel = clevels[clevelidx]; params.cdict = NULL; -- cgit v0.12 From e0d8add7917f5eafa4894979b5b68b0db1fea6f4 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 18 Apr 2018 15:38:41 -0400 Subject: Fix Framebench Output Buffer Sizing --- tests/framebench.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/framebench.c b/tests/framebench.c index 70cfec5..9752f23 100644 --- a/tests/framebench.c +++ b/tests/framebench.c @@ -347,6 +347,9 @@ int main(int argc, char *argv[]) { options.stableSrc = 1; out_size = LZ4F_compressFrameBound(in_size, &prefs); + if ((size_t)LZ4_compressBound(in_size) > out_size) { + out_size = LZ4_compressBound(in_size); + } out_buf = (char *)malloc(out_size); if (!out_buf) return 1; -- cgit v0.12 From e75153f508073d87e7087e8ef472879773f116f8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 19 Mar 2018 17:47:52 -0400 Subject: Add Debug Log Statements to HC --- lib/lz4frame.c | 21 +++++++++++++++++++++ lib/lz4hc.c | 18 ++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 507e4fe..0000023 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -95,6 +95,18 @@ You can contact the author at : #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif /*-************************************ * Basic Types @@ -457,6 +469,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); + DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict); if (!cdict) return NULL; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; @@ -479,6 +492,7 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) void LZ4F_freeCDict(LZ4F_CDict* cdict) { + DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict); if (cdict==NULL) return; /* support free on NULL */ FREEMEM(cdict->dictContent); LZ4_freeStream(cdict->fastCtx); @@ -530,6 +544,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp static void LZ4F_applyCDict(void* ctx, const LZ4F_CDict* cdict, int level) { + DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict); if (level < LZ4HC_CLEVEL_MIN) { LZ4_resetStream_fast((LZ4_stream_t *)ctx); LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); @@ -560,6 +575,8 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, BYTE* dstPtr = dstStart; BYTE* headerStart; + DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict); + if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); MEM_INIT(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; @@ -778,6 +795,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, LZ4F_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize); if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) @@ -916,6 +934,9 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, BYTE* dstPtr = dstStart; size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); + + DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr); + if (LZ4F_isError(flushSize)) return flushSize; dstPtr += flushSize; diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 111a09b..12d26c3 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -86,6 +86,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) **************************************/ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { + DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; @@ -641,6 +642,8 @@ static int LZ4HC_compress_generic ( { lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ }; + DEBUGLOG(4, "LZ4HC_compress_generic(%p, %p, %d)", ctx, src, *srcSizePtr); + if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ @@ -706,8 +709,14 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i * Streaming Functions **************************************/ /* allocation */ -LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { +LZ4_streamHC_t* LZ4_createStreamHC(void) { + LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); + DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr); + return LZ4_streamHCPtr; +} + +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ free(LZ4_streamHCPtr); return 0; @@ -718,6 +727,7 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } @@ -732,6 +742,7 @@ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLev int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize); if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; @@ -747,6 +758,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ @@ -764,6 +776,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, limitedOutput_directive limit) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_compressHC_continue_generic(%p, %p, %d)", LZ4_streamHCPtr, src, *srcSizePtr); /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); @@ -812,6 +825,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS { LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; -- cgit v0.12 From f895b9a6c69a059e62de8baca39b878ed1031418 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 19 Mar 2018 13:16:05 -0400 Subject: Add a Dictionary Context Pointer to the HC Context --- lib/lz4hc.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 7a25bee..1b08425 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -141,7 +141,8 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #include -typedef struct +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal { uint32_t hashTable[LZ4HC_HASHTABLESIZE]; uint16_t chainTable[LZ4HC_MAXD]; @@ -153,11 +154,13 @@ typedef struct uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ int compressionLevel; -} LZ4HC_CCtx_internal; + const LZ4HC_CCtx_internal* dictCtx; +}; #else -typedef struct +typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal; +struct LZ4HC_CCtx_internal { unsigned int hashTable[LZ4HC_HASHTABLESIZE]; unsigned short chainTable[LZ4HC_MAXD]; @@ -169,11 +172,12 @@ typedef struct unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ int compressionLevel; -} LZ4HC_CCtx_internal; + const LZ4HC_CCtx_internal* dictCtx; +}; #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From a992d11fc2204b7cb0a70962aa5911a58ed4918c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 21 Mar 2018 11:21:07 -0400 Subject: Fully Bounds Check Hash Table Reads --- lib/lz4hc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 12d26c3..6890613 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -222,7 +222,7 @@ LZ4HC_InsertAndGetWiderMatch ( DEBUGLOG(7, "First match at index %u / %u (lowLimit)", matchIndex, lowLimit); - while ((matchIndex>=lowLimit) && (nbAttempts)) { + while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) { DEBUGLOG(7, "remaining attempts : %i", nbAttempts); nbAttempts--; if (matchIndex >= dictLimit) { @@ -286,7 +286,7 @@ LZ4HC_InsertAndGetWiderMatch ( matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */ } } } } } - } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ + } /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */ return longest; } -- cgit v0.12 From fdeead0b09188213e51e037edcf9c9daf88d25c4 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 21 Mar 2018 14:49:47 -0400 Subject: Set dictCtx Rather than memcpy'ing Ctx --- lib/lz4frame.c | 9 +++------ lib/lz4hc.c | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 0000023..821cb49 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -549,12 +549,9 @@ static void LZ4F_applyCDict(void* ctx, LZ4_resetStream_fast((LZ4_stream_t *)ctx); LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); } else { - if (cdict) { - memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx)); - LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level); - } else { - LZ4_resetStreamHC((LZ4_streamHC_t*)(ctx), level); - } + LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse; + LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level); + internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL; } } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 6890613..16830c2 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -729,6 +729,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } -- cgit v0.12 From 6289ff4fb1a560115ffd2c5dff93389d917a6e4e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 21 Mar 2018 16:43:15 -0400 Subject: Call LZ4F_applyCDict Even on NULL CDict --- lib/lz4frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 821cb49..ea6a668 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -739,8 +739,8 @@ static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, in static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) { + LZ4F_applyCDict(ctx, cdict, level); if (cdict) { - LZ4F_applyCDict(ctx, cdict, level); return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); -- cgit v0.12 From 66d217e2400e07fd91753881890722b6dc28ee4b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 21 Mar 2018 16:54:36 -0400 Subject: Perform Lookups into the Dictionary Context --- lib/lz4hc.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 16830c2..2775713 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -203,6 +203,7 @@ LZ4HC_InsertAndGetWiderMatch ( { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; + const LZ4HC_CCtx_internal * const dictCtx = hc4->dictCtx; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; @@ -212,6 +213,7 @@ LZ4HC_InsertAndGetWiderMatch ( int nbAttempts = maxNbAttempts; U32 const pattern = LZ4_read32(ip); U32 matchIndex; + U32 dictMatchIndex; repeat_state_e repeat = rep_untested; size_t srcPatternLength = 0; @@ -288,6 +290,39 @@ LZ4HC_InsertAndGetWiderMatch ( } } } } } /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */ + if (dictCtx != NULL) { + ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; + dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + matchIndex = dictMatchIndex + dictIndexDelta; + while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictCtx->end - matchPtr); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) { + // mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit); + // } + back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + *matchpos = base + matchIndex + back; + *startpos = ip + back; + } + } + + { + U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } + } + } + return longest; } -- cgit v0.12 From 595ea582890031c30ded483130116ec6a0179a5c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 22 Mar 2018 16:59:50 -0400 Subject: Avoid Resetting Hash Table --- lib/lz4hc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 2775713..b6d9bf0 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; -- cgit v0.12 From b6c35ed6422f60a7a542ed237cadfd9dfe76c219 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 22 Mar 2018 17:00:04 -0400 Subject: Avoid Resetting Chain Table --- lib/lz4hc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b6d9bf0..0caf001 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -87,7 +87,6 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); - MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; hc4->end = start; -- cgit v0.12 From b88a0b4e8812777d1e715ab095738197f7f8eb96 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 26 Mar 2018 11:43:52 -0400 Subject: Only Perform Dict Lookup if Attempts Remain --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 0caf001..e005cd7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -288,7 +288,7 @@ LZ4HC_InsertAndGetWiderMatch ( } } } } } /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */ - if (dictCtx != NULL) { + if (dictCtx != NULL && nbAttempts) { ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + dictIndexDelta; -- cgit v0.12 From 4f7b7a8ffa3032a64aa173f7d8bfb7c258316154 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 28 Mar 2018 12:25:05 -0400 Subject: Clear Tables on Dict Load --- lib/lz4hc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e005cd7..85419f3 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -95,6 +95,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) hc4->lowLimit = 64 KB; } +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) +{ + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); +} + /* Update chains up to ip (excluded) */ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) @@ -782,6 +788,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + LZ4HC_clearTables (ctxPtr); ctxPtr->end = (const BYTE*)dictionary + dictSize; if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; -- cgit v0.12 From 22db704a73e38a869d846fa52ddcbb0a214a6c43 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 28 Mar 2018 12:26:54 -0400 Subject: Shift Dict Limit Checks out of the Loop --- lib/lz4hc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 85419f3..8ac650e 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -294,11 +294,11 @@ LZ4HC_InsertAndGetWiderMatch ( } } } } } /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */ - if (dictCtx != NULL && nbAttempts) { + if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + dictIndexDelta; - while (dictMatchIndex >= dictCtx->dictLimit && dictMatchIndex + dictCtx->base < dictCtx->end && dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { + while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { -- cgit v0.12 From 895e76cc20f366ea9f2999abc9ff3e0dd11b4222 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 4 Apr 2018 15:20:47 -0400 Subject: Push Previous Compression Offsets into the Past --- lib/lz4hc.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 8ac650e..c56d976 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -84,23 +84,29 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr) /************************************** * HC Compression **************************************/ -static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) -{ - DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); - hc4->nextToUpdate = 64 KB; - hc4->base = start - 64 KB; - hc4->end = start; - hc4->dictBase = start - 64 KB; - hc4->dictLimit = 64 KB; - hc4->lowLimit = 64 KB; -} - static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) { + DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4); MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); } +static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) +{ + U32 startingOffset = hc4->end - hc4->base + 64 KB; + DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); + if (startingOffset > 1 GB || startingOffset > (uptrval)start) { + LZ4HC_clearTables(hc4); + startingOffset = 64 KB; + } + hc4->nextToUpdate = startingOffset; + hc4->base = start - startingOffset; + hc4->end = start; + hc4->dictBase = start - startingOffset; + hc4->dictLimit = startingOffset; + hc4->lowLimit = startingOffset; +} + /* Update chains up to ip (excluded) */ LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) @@ -751,6 +757,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr); + LZ4_streamHCPtr->internal_donotuse.end = (void *)-1; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; return LZ4_streamHCPtr; } @@ -767,6 +776,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); -- cgit v0.12 From bdd7af6f71c235cea53912d0eec3b94555e9ffde Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 4 Apr 2018 15:59:00 -0400 Subject: Don't Bother Clearing Chain Table for Working Contexts --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index c56d976..673b4b3 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -96,7 +96,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) U32 startingOffset = hc4->end - hc4->base + 64 KB; DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); if (startingOffset > 1 GB || startingOffset > (uptrval)start) { - LZ4HC_clearTables(hc4); + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); startingOffset = 64 KB; } hc4->nextToUpdate = startingOffset; -- cgit v0.12 From a1beba13f7363df4b5d7ca9afe51963176df068d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 5 Apr 2018 16:32:26 -0400 Subject: Reset Stream in LZ4_compress_HC --- lib/lz4hc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 673b4b3..4a52f0c 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -732,7 +732,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif - int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); + int cSize; + statePtr->internal_donotuse.end = (void *)-1; + LZ4_resetStreamHC(statePtr, compressionLevel); + cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif -- cgit v0.12 From 221211d7d04478d74eec9fd76ccd98a73bd394ee Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 5 Apr 2018 16:32:40 -0400 Subject: Fix Offset Math --- lib/lz4hc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 4a52f0c..5479159 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -93,12 +93,13 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { - U32 startingOffset = hc4->end - hc4->base + 64 KB; + uptrval startingOffset = hc4->end - hc4->base; DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); if (startingOffset > 1 GB || startingOffset > (uptrval)start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); - startingOffset = 64 KB; + startingOffset = 0; } + startingOffset += MAX_DISTANCE; hc4->nextToUpdate = startingOffset; hc4->base = start - startingOffset; hc4->end = start; -- cgit v0.12 From 8f9a2db0e1ea492e836dd074dd0ea62720fb6169 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 6 Apr 2018 19:24:22 -0400 Subject: Fix Some Cast/Conversion Warnings --- lib/lz4hc.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5479159..11d820f 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -100,12 +100,12 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) startingOffset = 0; } startingOffset += MAX_DISTANCE; - hc4->nextToUpdate = startingOffset; + hc4->nextToUpdate = (U32) startingOffset; hc4->base = start - startingOffset; hc4->end = start; hc4->dictBase = start - startingOffset; - hc4->dictLimit = startingOffset; - hc4->lowLimit = startingOffset; + hc4->dictLimit = (U32) startingOffset; + hc4->lowLimit = (U32) startingOffset; } @@ -304,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch ( if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; - matchIndex = dictMatchIndex + dictIndexDelta; + matchIndex = dictMatchIndex + (int)dictIndexDelta; while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; @@ -314,9 +314,11 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE* vLimit = ip + (dictCtx->end - matchPtr); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; - // if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) { - // mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit); - // } + /* + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) { + mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit); + } + */ back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; mlt -= back; if (mlt > longest) { @@ -734,7 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in LZ4_streamHC_t* const statePtr = &state; #endif int cSize; - statePtr->internal_donotuse.end = (void *)-1; + statePtr->internal_donotuse.end = (const BYTE*)-1; LZ4_resetStreamHC(statePtr, compressionLevel); cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 @@ -761,7 +763,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr); - LZ4_streamHCPtr->internal_donotuse.end = (void *)-1; + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; return LZ4_streamHCPtr; -- cgit v0.12 From 8db291bc1dc95ab58904b6961a5ac0967b148952 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 5 Apr 2018 17:41:15 -0400 Subject: Remove Match Upper Bounds Check --- lib/lz4hc.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 11d820f..fa2cbb7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -235,7 +235,7 @@ LZ4HC_InsertAndGetWiderMatch ( DEBUGLOG(7, "First match at index %u / %u (lowLimit)", matchIndex, lowLimit); - while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) { + while ((matchIndex>=lowLimit) && (nbAttempts)) { DEBUGLOG(7, "remaining attempts : %i", nbAttempts); nbAttempts--; if (matchIndex >= dictLimit) { @@ -299,7 +299,7 @@ LZ4HC_InsertAndGetWiderMatch ( matchIndex -= (U32)backLength; /* let's go to farthest segment position, will find a match of length currentSegmentLength + maybe some back */ } } } } } - } /* while ((matchIndex>=lowLimit) && (matchIndex < (ip - base)) && (nbAttempts)) */ + } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; @@ -720,6 +720,7 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel); LZ4HC_init (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); @@ -735,10 +736,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif - int cSize; - statePtr->internal_donotuse.end = (const BYTE*)-1; - LZ4_resetStreamHC(statePtr, compressionLevel); - cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); + int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif @@ -750,6 +748,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + LZ4_resetStreamHC((LZ4_streamHC_t*)LZ4HC_Data, cLevel); LZ4HC_init(ctx, (const BYTE*) source); return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); } @@ -782,7 +781,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); - LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); -- cgit v0.12 From 3591fe8ab8b4391d1f9104b38b9ac19deac2a3e8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Apr 2018 15:09:59 -0400 Subject: Add Fast Reset Paths --- lib/lz4hc.c | 21 +++++++++++++++++++-- lib/lz4hc.h | 2 ++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index fa2cbb7..9deb90d 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -716,11 +716,11 @@ static int LZ4HC_compress_generic ( int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } -int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4_resetStreamHC((LZ4_streamHC_t*)state, compressionLevel); + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); LZ4HC_init (ctx, (const BYTE*)src); if (dstCapacity < LZ4_compressBound(srcSize)) return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); @@ -728,6 +728,13 @@ int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int src return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, noLimit); } +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ4_resetStreamHC ((LZ4_streamHC_t*)state, compressionLevel); + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} + int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) { #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 @@ -787,6 +794,16 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); } +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ + DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 1b08425..3b0d0d3 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -270,7 +270,9 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, */ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ -- cgit v0.12 From 61c7ceffede9607721ca496ac13d788625f9f668 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Apr 2018 15:25:48 -0400 Subject: Use Fast Reset API in LZ4F --- lib/lz4frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index ea6a668..d087a94 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -743,7 +743,7 @@ static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSi if (cdict) { return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); } - return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level); + return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level); } static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) -- cgit v0.12 From 0a2abacd909d0e1e1be9a82de6c2a693264f8afa Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Apr 2018 20:23:19 -0400 Subject: Use Fast Reset in LZ4F Again --- lib/lz4frame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index d087a94..a43f595 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -550,7 +550,7 @@ static void LZ4F_applyCDict(void* ctx, LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); } else { LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse; - LZ4_resetStreamHC((LZ4_streamHC_t*)ctx, level); + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level); internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL; } } -- cgit v0.12 From 22e16d5b509c56ad350ae42664ddcd3da7b97f1f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 18 Apr 2018 13:55:58 -0400 Subject: Split DictCtx-using Code Into Separate Inlining Chain --- lib/lz4hc.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 20 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 9deb90d..5c2b001 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -79,6 +79,8 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtx } dictCtx_directive; /************************************** @@ -210,7 +212,8 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts, - const int patternAnalysis) + const int patternAnalysis, + const dictCtx_directive dict) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; @@ -301,7 +304,7 @@ LZ4HC_InsertAndGetWiderMatch ( } } } } } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ - if (dictCtx != NULL && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { + if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + (int)dictIndexDelta; @@ -344,13 +347,14 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl const BYTE* const ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts, - const int patternAnalysis) + const int patternAnalysis, + const dictCtx_directive dict) { const BYTE* uselessPtr = ip; /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), * but this won't be the case here, as we define iLowLimit==ip, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ - return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis); + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, matchpos, &uselessPtr, maxNbAttempts, patternAnalysis, dict); } @@ -440,7 +444,8 @@ static int LZ4HC_compress_hashChain ( int* srcSizePtr, int const maxOutputSize, unsigned maxNbAttempts, - limitedOutput_directive limit + const limitedOutput_directive limit, + const dictCtx_directive dict ) { const int inputSize = *srcSizePtr; @@ -472,7 +477,7 @@ static int LZ4HC_compress_hashChain ( /* Main Loop */ while (ip <= mflimit) { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis); + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, &ref, maxNbAttempts, patternAnalysis, dict); if (mldictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} + +static int LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + assert(ctx->dictCtx != NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); +} + +static int LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } } @@ -1019,7 +1071,8 @@ typedef struct { LZ4_FORCE_INLINE LZ4HC_match_t LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, const BYTE* ip, const BYTE* const iHighLimit, - int minLen, int nbSearches) + int minLen, int nbSearches, + const dictCtx_directive dict) { LZ4HC_match_t match = { 0 , 0 }; const BYTE* matchPtr = NULL; @@ -1028,7 +1081,7 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ int const matchLength = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, &matchPtr, &ip, - nbSearches, 1 /* patternAnalysis */); + nbSearches, 1 /* patternAnalysis */, dict); if (matchLength <= minLen) return match; match.len = matchLength; match.off = (int)(ip-matchPtr); @@ -1044,8 +1097,9 @@ static int LZ4HC_compress_optimal ( int dstCapacity, int const nbSearches, size_t sufficient_len, - limitedOutput_directive limit, - int const fullUpdate + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict ) { #define TRAILING_LITERALS 3 @@ -1073,7 +1127,7 @@ static int LZ4HC_compress_optimal ( int best_mlen, best_off; int cur, last_match_pos = 0; - LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches); + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict); if (firstMatch.len==0) { ip++; continue; } if ((size_t)firstMatch.len > sufficient_len) { @@ -1143,10 +1197,10 @@ static int LZ4HC_compress_optimal ( DEBUGLOG(7, "search at rPos:%u", cur); if (fullUpdate) - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches); + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict); else /* only test matches of minimum length; slightly faster, but misses a few bytes */ - newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches); + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict); if (!newMatch.len) continue; if ( ((size_t)newMatch.len > sufficient_len) -- cgit v0.12 From b67de2a327644607c034250d105ba9374b0897e8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 18 Apr 2018 15:52:04 -0400 Subject: Force Inline on HashChain --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5c2b001..c050c59 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -437,7 +437,7 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( return 0; } -static int LZ4HC_compress_hashChain ( +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( LZ4HC_CCtx_internal* const ctx, const char* const source, char* const dest, -- cgit v0.12 From 0abc23f72e44dd9af86e7fb61a2497cee81ed320 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 19 Apr 2018 13:02:55 -0400 Subject: Copy DictCtx into Working Context on Inputs Larger than 4 KB --- lib/lz4hc.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index c050c59..c201559 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -720,6 +720,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal ( } } +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + static int LZ4HC_compress_generic_noDictCtx ( LZ4HC_CCtx_internal* const ctx, const char* const src, @@ -745,7 +747,16 @@ static int LZ4HC_compress_generic_dictCtx ( ) { assert(ctx->dictCtx != NULL); - return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); + if (*srcSizePtr > 4 KB) { + memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); + ctx->dictCtx = NULL; + return result; + } } static int LZ4HC_compress_generic ( -- cgit v0.12 From f4b13e17ea110498c41ba3ac8dc6df0438c37d29 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 19 Apr 2018 17:51:10 -0400 Subject: Don't Clear the Dictionary Context Until No Longer Useful --- lib/lz4hc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index c201559..9a02787 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -746,15 +746,18 @@ static int LZ4HC_compress_generic_dictCtx ( limitedOutput_directive limit ) { + size_t position = ctx->end - ctx->base - ctx->lowLimit; assert(ctx->dictCtx != NULL); - if (*srcSizePtr > 4 KB) { + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB) { memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); LZ4HC_setExternalDict(ctx, (const BYTE *)src); ctx->compressionLevel = cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); - ctx->dictCtx = NULL; return result; } } -- cgit v0.12 From 14c577d4c9332197dc0003833e043608b4b4b239 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 19 Apr 2018 19:36:34 -0400 Subject: Fix Signedness of Comparison --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 9a02787..20df8fa 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -308,7 +308,7 @@ LZ4HC_InsertAndGetWiderMatch ( ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + (int)dictIndexDelta; - while (dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { + while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { -- cgit v0.12 From 0064e8ebc7a475d27ba658ca92e40de75c9cac72 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 13:14:37 -0400 Subject: Remove Commented Out Support for Match Continuation over Segment Boundary --- lib/lz4hc.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 20df8fa..e09d8e0 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -317,11 +317,6 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE* vLimit = ip + (dictCtx->end - matchPtr); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; - /* - if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) { - mlt += LZ4_count(ip+mlt, base+lowLimit, iHighLimit); - } - */ back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; mlt -= back; if (mlt > longest) { -- cgit v0.12 From 8f118cf6e9f3030f52414177d93ab447e8e24128 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 14:08:06 -0400 Subject: Remove inputBuffer from Context, Work Around its Absence --- lib/lz4hc.c | 13 +++++++------ lib/lz4hc.h | 11 ++++++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e09d8e0..dff42f0 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -999,8 +999,8 @@ int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ + LZ4_resetStreamHC((LZ4_streamHC_t*)state, ((LZ4_streamHC_t*)state)->internal_donotuse.compressionLevel); LZ4HC_init(ctx, (const BYTE*)inputBuffer); - ctx->inputBuffer = inputBuffer; return 0; } @@ -1008,9 +1008,8 @@ void* LZ4_createHC (const char* inputBuffer) { LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4_resetStreamHC(hc4, 0 /* compressionLevel */); LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); - assert(sizeof(size_t) == sizeof(void*)); - hc4->internal_donotuse.inputBuffer = (void*)(size_t)inputBuffer; /* ugly hack, circumvent -Wcast-qual */ return hc4; } @@ -1032,9 +1031,11 @@ int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, c char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { - LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); - return (char*)(hc4->inputBuffer) + dictSize; + LZ4_streamHC_t *ctx = (LZ4_streamHC_t*)LZ4HC_Data; + const BYTE *bufferStart = ctx->internal_donotuse.base + ctx->internal_donotuse.lowLimit; + LZ4_resetStreamHC_fast(ctx, ctx->internal_donotuse.compressionLevel); + /* avoid const char * -> char * conversion warning :( */ + return (char *)(uptrval)bufferStart; } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 3b0d0d3..cfc5d9e 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -149,7 +149,6 @@ struct LZ4HC_CCtx_internal const uint8_t* end; /* next block here to continue on current prefix */ const uint8_t* base; /* All index relative to this position */ const uint8_t* dictBase; /* alternate base for extDict */ - void* inputBuffer; /* deprecated */ uint32_t dictLimit; /* below that point, need extDict */ uint32_t lowLimit; /* below that point, no more dict */ uint32_t nextToUpdate; /* index from which to continue dictionary update */ @@ -167,7 +166,6 @@ struct LZ4HC_CCtx_internal const unsigned char* end; /* next block here to continue on current prefix */ const unsigned char* base; /* All index relative to this position */ const unsigned char* dictBase; /* alternate base for extDict */ - void* inputBuffer; /* deprecated */ unsigned int dictLimit; /* below that point, need extDict */ unsigned int lowLimit; /* below that point, no more dict */ unsigned int nextToUpdate; /* index from which to continue dictionary update */ @@ -210,7 +208,14 @@ LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_co LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Deprecated Streaming functions; should no longer be used */ +/* Obsolete streaming functions; degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, use of + * LZ4_slideInputBufferHC() will truncate the history of the stream, rather + * than preserve a window-sized chunk of history. + */ LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data); -- cgit v0.12 From ca833f928f64d0b88fbfd2a8a8bc11b8333e487d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 14:16:27 -0400 Subject: Also Reset the Chain Table --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index dff42f0..124da86 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) uptrval startingOffset = hc4->end - hc4->base; DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); if (startingOffset > 1 GB || startingOffset > (uptrval)start) { - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + LZ4HC_clearTables(hc4); startingOffset = 0; } startingOffset += MAX_DISTANCE; -- cgit v0.12 From d7347f9eeaffe5351886b6174f9974bd541b9b7d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 14:52:13 -0400 Subject: Add API for Attaching Dictionaries --- lib/lz4frame.c | 3 +-- lib/lz4hc.c | 3 +++ lib/lz4hc.h | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index a43f595..d4d9397 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -549,9 +549,8 @@ static void LZ4F_applyCDict(void* ctx, LZ4_resetStream_fast((LZ4_stream_t *)ctx); LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); } else { - LZ4HC_CCtx_internal *internal_ctx = &((LZ4_streamHC_t *)ctx)->internal_donotuse; LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level); - internal_ctx->dictCtx = cdict ? &(cdict->HCCtx->internal_donotuse) : NULL; + LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL); } } diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 124da86..690015f 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -887,6 +887,9 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int return dictSize; } +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} /* compression */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index cfc5d9e..fb8397e 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -279,5 +279,32 @@ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLeve int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); +/*! LZ4_attach_HC_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_API void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream); + #endif /* LZ4_HC_SLO_098092834 */ #endif /* LZ4_HC_STATIC_LINKING_ONLY */ -- cgit v0.12 From 3f087cf1cbcb0e0082ead149a621d259cb57e7ba Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 15:00:53 -0400 Subject: Add Comments on New Public APIs --- lib/lz4hc.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index fb8397e..a762d3f 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -275,8 +275,37 @@ int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, */ void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +/*! LZ4_resetStreamHC_fast() : + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + */ void LZ4_resetStreamHC_fast(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); /*! LZ4_attach_HC_dictionary() : -- cgit v0.12 From 209c9c29d1baf5b2f4fb1fc38b2612dd95d1b9a1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 15:16:41 -0400 Subject: Add Some Simple Fuzzer Tests --- tests/fuzzer.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 244cc4f..8b21f8e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -446,7 +446,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c /* Test compression HC using external state */ FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()"); ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); - FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed"); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed") + + /* Test compression HC using fast reset external state */ + FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()"); + ret = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel); + FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC_fastReset() failed"); /* Test compression using external state */ FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()"); @@ -810,6 +815,49 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); } + /* Compress HC using external dictionary stream */ + FUZ_DISPLAYTEST(); + { + LZ4_streamHC_t LZ4_streamHC; + + LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); + LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); + LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); + blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed"); + + FUZ_DISPLAYTEST(); + LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); + ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize); + + FUZ_DISPLAYTEST(); + LZ4_resetStreamHC (&LZ4_streamHC, compressionLevel); + LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); + ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer"); + + FUZ_DISPLAYTEST(); + LZ4_resetStreamHC_fast (&LZ4_streamHC, compressionLevel); + LZ4_attach_HC_dictionary(&LZ4_streamHC, &LZ4dictHC); + ret = LZ4_compress_HC_continue(&LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer"); + + FUZ_DISPLAYTEST(); + decodedBuffer[blockSize] = 0; + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size"); + { U32 const crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); + } + } + /* Compress HC continue destSize */ FUZ_DISPLAYTEST(); { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5; -- cgit v0.12 From 7874cf06b3307a7ba5efdd141d068887480aaf11 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 15:30:08 -0400 Subject: Consts and Asserts and Other Minor Nits --- lib/lz4hc.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 690015f..e5eb11d 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -305,10 +305,13 @@ LZ4HC_InsertAndGetWiderMatch ( } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { - ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; + const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; + /* bounds check, since we need to downcast */ + assert(dictIndexDelta <= 1 GB); + assert(dictIndexDelta >= -1 GB); dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; matchIndex = dictMatchIndex + (int)dictIndexDelta; - while ((ptrdiff_t) dictMatchIndex + MAX_DISTANCE > ip - base - dictIndexDelta && nbAttempts--) { + while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { @@ -741,7 +744,7 @@ static int LZ4HC_compress_generic_dictCtx ( limitedOutput_directive limit ) { - size_t position = ctx->end - ctx->base - ctx->lowLimit; + const size_t position = ctx->end - ctx->base - ctx->lowLimit; assert(ctx->dictCtx != NULL); if (position >= 64 KB) { ctx->dictCtx = NULL; @@ -752,8 +755,7 @@ static int LZ4HC_compress_generic_dictCtx ( ctx->compressionLevel = cLevel; return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); } else { - int result = LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); - return result; + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtx); } } @@ -804,7 +806,7 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in LZ4_streamHC_t state; LZ4_streamHC_t* const statePtr = &state; #endif - int cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); + int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 free(statePtr); #endif -- cgit v0.12 From 1d2500d44e97a46eb447745d02652e02435baadb Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 17:10:47 -0400 Subject: Handle Index Underflows Safely --- lib/lz4hc.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index e5eb11d..843b539 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -221,7 +221,8 @@ LZ4HC_InsertAndGetWiderMatch ( const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - MAX_DISTANCE; + const U32 ipIndex = (U32)(ip - base); + const U32 lowLimit = (hc4->lowLimit + 64 KB > ipIndex) ? hc4->lowLimit : ipIndex - MAX_DISTANCE; const BYTE* const dictBase = hc4->dictBase; int const delta = (int)(ip-iLowLimit); int nbAttempts = maxNbAttempts; @@ -304,14 +305,12 @@ LZ4HC_InsertAndGetWiderMatch ( } } } } } /* while ((matchIndex>=lowLimit) && (nbAttempts)) */ - if (dict == usingDictCtx && nbAttempts && ip - base - lowLimit < MAX_DISTANCE) { - const ptrdiff_t dictIndexDelta = dictCtx->base - dictCtx->end + lowLimit; - /* bounds check, since we need to downcast */ - assert(dictIndexDelta <= 1 GB); - assert(dictIndexDelta >= -1 GB); + if (dict == usingDictCtx && nbAttempts && ipIndex - lowLimit < MAX_DISTANCE) { + size_t const dictEndOffset = dictCtx->end - dictCtx->base; + assert(dictEndOffset <= 1 GB); dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; - matchIndex = dictMatchIndex + (int)dictIndexDelta; - while ((ptrdiff_t) matchIndex + MAX_DISTANCE > ip - base && nbAttempts--) { + matchIndex = dictMatchIndex + lowLimit - (U32)dictEndOffset; + while (ipIndex - matchIndex <= MAX_DISTANCE && nbAttempts--) { const BYTE* const matchPtr = dictCtx->base + dictMatchIndex; if (LZ4_read32(matchPtr) == pattern) { -- cgit v0.12 From 86b381e40b6ec3394582a2113c7bc80f1d619bab Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 17:13:40 -0400 Subject: Fix Constant Value --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 843b539..48c42c6 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -101,7 +101,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) LZ4HC_clearTables(hc4); startingOffset = 0; } - startingOffset += MAX_DISTANCE; + startingOffset += 64 KB; hc4->nextToUpdate = (U32) startingOffset; hc4->base = start - startingOffset; hc4->end = start; -- cgit v0.12 From 756ed402da8161c7bc3749bebaac069259312d8b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 17:56:26 -0400 Subject: Sign-Extend -1 to Pointer Width --- lib/lz4hc.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 48c42c6..a973086 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -830,8 +830,7 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); - DEBUGLOG(4, "LZ4_createStreamHC() -> %p", LZ4_streamHCPtr); - LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1; + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; return LZ4_streamHCPtr; @@ -850,7 +849,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ DEBUGLOG(4, "LZ4_resetStreamHC(%p, %d)", LZ4_streamHCPtr, compressionLevel); - LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)-1; + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); -- cgit v0.12 From 85cac61dd8823886132b4a9c2ff4a43b237da917 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 19:35:51 -0400 Subject: Don't Segfault on Malloc Failure --- lib/lz4hc.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index a973086..b0325a7 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -830,9 +830,11 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); - LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; - LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + if (LZ4_streamHCPtr != NULL) { + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; + } return LZ4_streamHCPtr; } -- cgit v0.12 From a8cb2feffdead7b49a09aad0dc7b13dcff206e5b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 19:37:07 -0400 Subject: Tolerate Base Pointer Underflow --- lib/lz4hc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b0325a7..b70d03b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -97,7 +97,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { uptrval startingOffset = hc4->end - hc4->base; DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); - if (startingOffset > 1 GB || startingOffset > (uptrval)start) { + if (startingOffset > 1 GB) { LZ4HC_clearTables(hc4); startingOffset = 0; } @@ -898,7 +898,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); - if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; -- cgit v0.12 From fcc99d1f31bf8a913fa91a5455524b4a14b9a03d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 19:37:28 -0400 Subject: Simpler loadDict() Reset --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index b70d03b..23d84f4 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -882,8 +882,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictionary += dictSize - 64 KB; dictSize = 64 KB; } + LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel); LZ4HC_init (ctxPtr, (const BYTE*)dictionary); - LZ4HC_clearTables (ctxPtr); ctxPtr->end = (const BYTE*)dictionary + dictSize; if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; -- cgit v0.12 From 1895fa19a4f04b395f9ebb1fe38049ab4c909fc3 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 20:14:12 -0400 Subject: Remove Redundant Static Assert --- lib/lz4hc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 23d84f4..f91ef4a 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -859,7 +859,6 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) { - LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ DEBUGLOG(4, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); LZ4_streamHCPtr->internal_donotuse.end -= (uptrval)LZ4_streamHCPtr->internal_donotuse.base; LZ4_streamHCPtr->internal_donotuse.base = NULL; -- cgit v0.12 From ee67f25576f111972bd5003c527d1ebee598dc71 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Apr 2018 20:18:30 -0400 Subject: Change vLimit Calculation --- lib/lz4hc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index f91ef4a..a12f298 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -316,7 +316,7 @@ LZ4HC_InsertAndGetWiderMatch ( if (LZ4_read32(matchPtr) == pattern) { int mlt; int back = 0; - const BYTE* vLimit = ip + (dictCtx->end - matchPtr); + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; back = delta ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->base + dictCtx->dictLimit) : 0; -- cgit v0.12 From 13271a88d7c21bee4ee61f619ef359afc707f23c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 24 Apr 2018 11:55:53 -0400 Subject: Revert Stream Size Const to Correct Value --- lib/lz4hc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4hc.h b/lib/lz4hc.h index a762d3f..28e2528 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -175,7 +175,7 @@ struct LZ4HC_CCtx_internal #endif -#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 64) /* 262200 */ +#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 262200 */ #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) union LZ4_streamHC_u { size_t table[LZ4_STREAMHCSIZE_SIZET]; -- cgit v0.12 From db9deb7b747d630a5778ed588d88dae82328dc62 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 19 Apr 2018 18:56:01 -0400 Subject: Remove the Framebench Tool --- tests/Makefile | 6 +- tests/framebench.c | 412 ----------------------------------------------------- 2 files changed, 1 insertion(+), 417 deletions(-) delete mode 100644 tests/framebench.c diff --git a/tests/Makefile b/tests/Makefile index f6a4ff3..2b93c9f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -63,7 +63,7 @@ NB_LOOPS ?= -i1 default: all -all: fullbench fuzzer frametest datagen framebench +all: fullbench fuzzer frametest datagen all32: CFLAGS+=-m32 all32: all @@ -99,9 +99,6 @@ fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -framebench: lz4frame.o lz4.o lz4hc.o xxhash.o framebench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - datagen : $(PRGDIR)/datagen.c datagencli.c $(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT) @@ -113,7 +110,6 @@ clean: fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ frametest$(EXT) frametest32$(EXT) \ - framebench$(EXT) \ fasttest$(EXT) datagen$(EXT) checkTag$(EXT) @rm -fR $(TESTDIR) @echo Cleaning completed diff --git a/tests/framebench.c b/tests/framebench.c deleted file mode 100644 index 9752f23..0000000 --- a/tests/framebench.c +++ /dev/null @@ -1,412 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define LZ4_STATIC_LINKING_ONLY -#include "lz4.h" -#include "lz4hc.h" -#include "lz4frame.h" -#include "lz4frame_static.h" - -#define LZ4F_CHECK(x) { typeof(x) _x = (x); if (LZ4F_isError(_x)) { fprintf(stderr, "Error!: %s\n", LZ4F_getErrorName(_x)); return 0; } } - -typedef struct { - const char *run_name; - size_t iter; - LZ4_stream_t *ctx; - LZ4_streamHC_t *hcctx; - LZ4F_cctx *cctx; - LZ4F_dctx *dctx; - const char *dictbuf; - size_t dictsize; - char *obuf; - size_t osize; - const char* ibuf; - const char* isample; - size_t isize; - size_t num_ibuf; - char *checkbuf; - size_t checksize; - int clevel; - const LZ4F_CDict* cdict; - LZ4F_preferences_t* prefs; - const LZ4F_compressOptions_t* options; -} bench_params_t; - -size_t compress_frame(bench_params_t *p) { - LZ4F_cctx *cctx = p->cctx; - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - const LZ4F_CDict* cdict = p->cdict; - LZ4F_preferences_t* prefs = p->prefs; - - size_t oused; - - prefs->frameInfo.contentSize = isize; - - oused = LZ4F_compressFrame_usingCDict( - cctx, - obuf, - osize, - isample, - isize, - cdict, - prefs); - LZ4F_CHECK(oused); - - return oused; -} - -size_t compress_begin(bench_params_t *p) { - LZ4F_cctx *cctx = p->cctx; - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - const LZ4F_CDict* cdict = p->cdict; - LZ4F_preferences_t* prefs = p->prefs; - const LZ4F_compressOptions_t* options = p->options; - - char *oend = obuf + osize; - size_t oused; - - prefs->frameInfo.contentSize = isize; - - oused = LZ4F_compressBegin_usingCDict(cctx, obuf, oend - obuf, cdict, prefs); - LZ4F_CHECK(oused); - obuf += oused; - oused = LZ4F_compressUpdate( - cctx, - obuf, - oend - obuf, - isample, - isize, - options); - LZ4F_CHECK(oused); - obuf += oused; - oused = LZ4F_compressEnd(cctx, obuf, oend - obuf, options); - LZ4F_CHECK(oused); - - return obuf - p->obuf; -} - -size_t compress_default(bench_params_t *p) { - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - - char *oend = obuf + osize; - size_t oused; - - oused = LZ4_compress_default(isample, obuf, isize, oend - obuf); - obuf += oused; - - return obuf - p->obuf; -} - -size_t compress_extState(bench_params_t *p) { - LZ4_stream_t *ctx = p->ctx; - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - int clevel = p->clevel; - - char *oend = obuf + osize; - size_t oused; - - oused = LZ4_compress_fast_extState_fastReset( - ctx, isample, obuf, isize, oend - obuf, clevel); - obuf += oused; - - return obuf - p->obuf; -} - -size_t compress_hc(bench_params_t *p) { - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - int clevel = p->clevel; - - char *oend = obuf + osize; - size_t oused; - - oused = LZ4_compress_HC( - isample, obuf, isize, oend - obuf, clevel); - obuf += oused; - - return obuf - p->obuf; -} - -size_t compress_hc_extState(bench_params_t *p) { - LZ4_streamHC_t *hcctx = p->hcctx; - char *obuf = p->obuf; - size_t osize = p->osize; - const char* isample = p->isample; - size_t isize = p->isize; - int clevel = p->clevel; - - char *oend = obuf + osize; - size_t oused; - - oused = LZ4_compress_HC_extStateHC( - hcctx, - isample, obuf, - isize, oend - obuf, clevel); - obuf += oused; - - return obuf - p->obuf; -} - -size_t check_lz4(bench_params_t *p, size_t csize) { - (void)csize; - memset(p->checkbuf, 0xFF, p->checksize); - return LZ4_decompress_fast_usingDict(p->obuf, p->checkbuf, p->isize, - p->dictbuf, p->dictsize) - && !memcmp(p->isample, p->checkbuf, p->isize); -} - -size_t check_lz4f(bench_params_t *p, size_t csize) { - size_t cp = 0; - size_t dp = 0; - size_t dsize = p->checksize; - size_t cleft = csize; - size_t dleft = dsize; - size_t ret; - memset(p->checkbuf, 0xFF, p->checksize); - LZ4F_resetDecompressionContext(p->dctx); - do { - ret = LZ4F_decompress_usingDict( - p->dctx, p->checkbuf + dp, &dleft, p->obuf + cp, &cleft, - p->dictbuf, p->dictsize, NULL); - cp += cleft; - dp += dleft; - cleft = csize - cp; - dleft = dsize - dp; - if (LZ4F_isError(ret)) return 0; - } while (cleft); - return !memcmp(p->isample, p->checkbuf, p->isize); -} - - -uint64_t bench( - const char *bench_name, - size_t (*fun)(bench_params_t *), - size_t (*checkfun)(bench_params_t *, size_t), - bench_params_t *params -) { - struct timespec start, end; - size_t i, osize = 0, o = 0; - size_t time_taken = 0; - uint64_t total_repetitions = 0; - uint64_t repetitions = 2; - - if (clock_gettime(CLOCK_MONOTONIC_RAW, &start)) return 0; - - while (time_taken < 25 * 1000 * 1000) { // benchmark over at least 1ms - if (total_repetitions) { - repetitions = total_repetitions; // double previous - } - - for (i = 0; i < repetitions; i++) { - params->iter = i; - if (params->num_ibuf == 1) { - params->isample = params->ibuf; - } else { - params->isample = params->ibuf + ((i * 2654435761U) % ((params->num_ibuf - 1) * params->isize)); - } - o = fun(params); - if (!o) { - fprintf( - stderr, - "%-19s: %-30s @ lvl %2d: %8ld B: FAILED!\n", - params->run_name, bench_name, params->clevel, - params->isize); - return 0; - } - osize += o; - } - - if (clock_gettime(CLOCK_MONOTONIC_RAW, &end)) return 0; - - time_taken = (1000 * 1000 * 1000 * end.tv_sec + end.tv_nsec) - - (1000 * 1000 * 1000 * start.tv_sec + start.tv_nsec); - total_repetitions += repetitions; - } - - o = checkfun(params, o); - if (!o) { - fprintf( - stderr, - "%-19s: %-30s @ lvl %2d: %8ld B: CHECK FAILED!\n", - params->run_name, bench_name, params->clevel, - params->isize); - return 0; - } - - fprintf( - stderr, - "%-19s: %-30s @ lvl %2d: %8ld B -> %8ld B, %6ld iters, %10ld ns, %10ld ns/iter, %7.2lf MB/s\n", - params->run_name, bench_name, params->clevel, - params->isize, osize / total_repetitions, - total_repetitions, time_taken, time_taken / total_repetitions, - ((double) 1000 * params->isize * total_repetitions) / time_taken - ); - - return time_taken; -} - -int main(int argc, char *argv[]) { - char *run_name; - - struct stat st; - size_t bytes_read; - - const char *dict_fn; - size_t dict_size; - char *dict_buf; - FILE *dict_file; - - const char *in_fn; - size_t in_size; - size_t num_in_buf; - size_t cur_in_buf; - char *in_buf; - FILE *in_file; - - size_t out_size; - char *out_buf; - - size_t check_size; - char *check_buf; - - LZ4_stream_t *ctx; - LZ4_streamHC_t *hcctx; - LZ4F_cctx *cctx; - LZ4F_dctx *dctx; - LZ4F_CDict *cdict; - LZ4F_preferences_t prefs; - LZ4F_compressOptions_t options; - - int clevels[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; - unsigned int clevelidx; - - bench_params_t params; - - if (argc != 4) return 1; - run_name = argv[1]; - dict_fn = argv[2]; - in_fn = argv[3]; - - if (stat(dict_fn, &st)) return 1; - dict_size = st.st_size; - dict_buf = (char *)malloc(dict_size); - if (!dict_buf) return 1; - dict_file = fopen(dict_fn, "r"); - bytes_read = fread(dict_buf, 1, dict_size, dict_file); - if (bytes_read != dict_size) return 1; - - if (stat(in_fn, &st)) return 1; - in_size = st.st_size; - num_in_buf = 256 * 1024 * 1024 / in_size; - if (num_in_buf == 0) { - num_in_buf = 1; - } - - in_buf = (char *)malloc(in_size * num_in_buf); - if (!in_buf) return 1; - in_file = fopen(in_fn, "r"); - bytes_read = fread(in_buf, 1, in_size, in_file); - if (bytes_read != in_size) return 1; - - for(cur_in_buf = 1; cur_in_buf < num_in_buf; cur_in_buf++) { - memcpy(in_buf + cur_in_buf * in_size, in_buf, in_size); - } - - check_size = in_size; - check_buf = (char *)malloc(check_size); - if (!check_buf) return 1; - - memset(&prefs, 0, sizeof(prefs)); - prefs.autoFlush = 1; - if (in_size < 64 * 1024) - prefs.frameInfo.blockMode = LZ4F_blockIndependent; - prefs.frameInfo.contentSize = in_size; - - memset(&options, 0, sizeof(options)); - options.stableSrc = 1; - - out_size = LZ4F_compressFrameBound(in_size, &prefs); - if ((size_t)LZ4_compressBound(in_size) > out_size) { - out_size = LZ4_compressBound(in_size); - } - out_buf = (char *)malloc(out_size); - if (!out_buf) return 1; - - if (LZ4F_isError(LZ4F_createCompressionContext(&cctx, LZ4F_VERSION))) return 1; - if (cctx == NULL) return 1; - - if (LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION))) return 1; - if (cctx == NULL) return 1; - - ctx = LZ4_createStream(); - if (ctx == NULL) return 1; - - hcctx = LZ4_createStreamHC(); - if (hcctx == NULL) return 1; - - cdict = LZ4F_createCDict(dict_buf, dict_size); - if (!cdict) return 1; - - fprintf(stderr, "dict size: %zd\n", dict_size); - fprintf(stderr, "input size: %zd\n", in_size); - - params.run_name = run_name; - params.ctx = ctx; - params.hcctx = hcctx; - params.cctx = cctx; - params.dctx = dctx; - params.dictbuf = dict_buf; - params.dictsize = dict_size; - params.obuf = out_buf; - params.osize = out_size; - params.ibuf = in_buf; - params.isize = in_size; - params.num_ibuf = num_in_buf; - params.checkbuf = check_buf; - params.checksize = check_size; - params.clevel = 1; - params.cdict = NULL; - params.prefs = &prefs; - params.options = &options; - - for (clevelidx = 0; clevelidx < sizeof(clevels) / sizeof(clevels[0]); clevelidx++) { - params.clevel = clevels[clevelidx]; - params.prefs->compressionLevel = clevels[clevelidx]; - params.cdict = NULL; - - bench("LZ4_compress_default" , compress_default , check_lz4 , ¶ms); - bench("LZ4_compress_fast_extState" , compress_extState , check_lz4 , ¶ms); - bench("LZ4_compress_HC" , compress_hc , check_lz4 , ¶ms); - bench("LZ4_compress_HC_extStateHC" , compress_hc_extState, check_lz4 , ¶ms); - bench("LZ4F_compressFrame" , compress_frame , check_lz4f, ¶ms); - bench("LZ4F_compressBegin" , compress_begin , check_lz4f, ¶ms); - - params.cdict = cdict; - - bench("LZ4F_compressFrame_usingCDict", compress_frame , check_lz4f, ¶ms); - bench("LZ4F_compressBegin_usingCDict", compress_begin , check_lz4f, ¶ms); - } - - return 0; -} -- cgit v0.12 From 5ed1463bf4668a0e668679ae1e458aa73aa9a6ec Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 19 Apr 2018 21:00:19 -0400 Subject: Remove Debug Log Statements --- lib/lz4frame.c | 21 --------------------- lib/lz4hc.c | 14 +++++--------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index d4d9397..844c8d1 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -95,18 +95,6 @@ You can contact the author at : #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) -# include -static int g_debuglog_enable = 1; -# define DEBUGLOG(l, ...) { \ - if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ -#endif /*-************************************ * Basic Types @@ -469,7 +457,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) { const char* dictStart = (const char*)dictBuffer; LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict)); - DEBUGLOG(4, "LZ4F_createCDict(%p) -> %p", dictBuffer, cdict); if (!cdict) return NULL; if (dictSize > 64 KB) { dictStart += dictSize - 64 KB; @@ -492,7 +479,6 @@ LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) void LZ4F_freeCDict(LZ4F_CDict* cdict) { - DEBUGLOG(4, "LZ4F_freeCDict(%p)", cdict); if (cdict==NULL) return; /* support free on NULL */ FREEMEM(cdict->dictContent); LZ4_freeStream(cdict->fastCtx); @@ -544,7 +530,6 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp static void LZ4F_applyCDict(void* ctx, const LZ4F_CDict* cdict, int level) { - DEBUGLOG(5, "LZ4F_applyCDict(%p, %p)", ctx, cdict); if (level < LZ4HC_CLEVEL_MIN) { LZ4_resetStream_fast((LZ4_stream_t *)ctx); LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL); @@ -571,8 +556,6 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr, BYTE* dstPtr = dstStart; BYTE* headerStart; - DEBUGLOG(4, "LZ4F_compressBegin_usingCDict(%p, %p)", cctxPtr, cdict); - if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); MEM_INIT(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; @@ -791,7 +774,6 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, LZ4F_lastBlockStatus lastBlockCompressed = notDone; compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); - DEBUGLOG(4, "LZ4F_compressUpdate(%p, %p, %zd)", cctxPtr, srcBuffer, srcSize); if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC); if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) @@ -930,9 +912,6 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, BYTE* dstPtr = dstStart; size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr); - - DEBUGLOG(4, "LZ4F_compressEnd(%p)", cctxPtr); - if (LZ4F_isError(flushSize)) return flushSize; dstPtr += flushSize; diff --git a/lib/lz4hc.c b/lib/lz4hc.c index a12f298..4126ef8 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -88,7 +88,6 @@ typedef enum { noDictCtx, usingDictCtx } dictCtx_directive; **************************************/ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) { - DEBUGLOG(4, "LZ4HC_clearTables(%p)", hc4); MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); } @@ -96,7 +95,6 @@ static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) { uptrval startingOffset = hc4->end - hc4->base; - DEBUGLOG(4, "LZ4HC_init(%p, %p)", hc4, start); if (startingOffset > 1 GB) { LZ4HC_clearTables(hc4); startingOffset = 0; @@ -830,11 +828,9 @@ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, i /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { LZ4_streamHC_t* LZ4_streamHCPtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); - if (LZ4_streamHCPtr != NULL) { - LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; - LZ4_streamHCPtr->internal_donotuse.base = NULL; - LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; - } + LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1; + LZ4_streamHCPtr->internal_donotuse.base = NULL; + LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL; return LZ4_streamHCPtr; } @@ -881,8 +877,8 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictionary += dictSize - 64 KB; dictSize = 64 KB; } - LZ4_resetStreamHC(LZ4_streamHCPtr, LZ4_streamHCPtr->internal_donotuse.compressionLevel); LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + LZ4HC_clearTables (ctxPtr); ctxPtr->end = (const BYTE*)dictionary + dictSize; if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); return dictSize; @@ -897,7 +893,7 @@ void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) { DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); - if (ctxPtr->end - ctxPtr->base - ctxPtr->nextToUpdate >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; -- cgit v0.12