From 43465ae8ff2ff07020ef6183172c4160ec6415fb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Aug 2014 14:27:22 +0100 Subject: Added : examples from Takayuki Matsuoka --- examples/Makefile | 85 ++++++++++++++ examples/blockStreaming_doubleBuffer.c | 195 +++++++++++++++++++++++++++++++ examples/blockStreaming_lineByLine.c | 207 +++++++++++++++++++++++++++++++++ examples/blockStreaming_ringBuffer.c | 186 +++++++++++++++++++++++++++++ examples/printVersion.c | 7 +- 5 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 examples/Makefile create mode 100644 examples/blockStreaming_doubleBuffer.c create mode 100644 examples/blockStreaming_lineByLine.c create mode 100644 examples/blockStreaming_ringBuffer.c diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000..5704c55 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,85 @@ +# ########################################################################## +# LZ4 examples - Makefile +# Copyright (C) Yann Collet 2011-2014 +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - LZ4 source repository : http://code.google.com/p/lz4/ +# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c +# ########################################################################## +# lz4 : Command Line Utility, supporting gzip-like arguments +# lz4c : CLU, supporting also legacy lz4demo arguments +# lz4c32: Same as lz4c, but forced to compile in 32-bits mode +# fuzzer : Test tool, to check lz4 integrity on target platform +# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# fullbench : Precisely measure speed for each LZ4 function variant +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# ########################################################################## + +CC := $(CC) +CFLAGS ?= -O3 +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug +FLAGS = -I.. $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) + +TESTFILE= Makefile +LZ4DIR=.. + + +# Minimize test target for Travis CI's Build Matrix +ifeq ($(LZ4_TRAVIS_CI_ENV),-m32) +CFLAGS += -m32 +else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64) +endif + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +VOID = nul +else +EXT = +VOID = /dev/null +endif + + +default: all + +all: printVersion doubleBuffer ringBuffer lineCompress + +printVersion: $(LZ4DIR)/lz4.c printVersion.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +test : all + ./printVersion$(EXT) + ./doubleBuffer$(EXT) $(TESTFILE) + ./ringBuffer$(EXT) $(TESTFILE) + ./lineCompress$(EXT) $(TESTFILE) + +clean: + @rm -f core *.o *.dec *-0 *-8192 *.lz4s \ + printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT) + @echo Cleaning completed + diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c new file mode 100644 index 0000000..bd87e77 --- /dev/null +++ b/examples/blockStreaming_doubleBuffer.c @@ -0,0 +1,195 @@ +// LZ4 streaming API example : double buffer +// Copyright : Takayuki Matsuoka + + +#define _CRT_SECURE_NO_WARNINGS // for MSVC +#include "lz4.h" + +#include +#include +#include +#include + +enum { + BLOCK_BYTES = 1024 * 8, +// BLOCK_BYTES = 1024 * 64, +}; + + +size_t write_int(FILE* fp, int i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int(FILE* fp, int* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, size_t arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + LZ4_stream_t lz4Stream_body = { 0 }; + LZ4_stream_t* lz4Stream = &lz4Stream_body; + + char inpBuf[2][BLOCK_BYTES]; + int inpBufIndex = 0; + + for(;;) { + char* const inpPtr = inpBuf[inpBufIndex]; + const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES); + if(0 == inpBytes) { + break; + } + + { + char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; + const int cmpBytes = LZ4_compress_continue( + lz4Stream, inpPtr, cmpBuf, inpBytes); + if(cmpBytes <= 0) { + break; + } + write_int(outFp, cmpBytes); + write_bin(outFp, cmpBuf, (size_t) cmpBytes); + } + + inpBufIndex = (inpBufIndex + 1) % 2; + } + + write_int(outFp, 0); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; + + char decBuf[2][BLOCK_BYTES]; + int decBufIndex = 0; + + for(;;) { + char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; + int cmpBytes = 0; + + { + const size_t readCount0 = read_int(inpFp, &cmpBytes); + if(readCount0 != 1 || cmpBytes <= 0) { + break; + } + + const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes); + if(readCount1 != (size_t) cmpBytes) { + break; + } + } + + { + char* const decPtr = decBuf[decBufIndex]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES); + if(decBytes <= 0) { + break; + } + write_bin(outFp, decPtr, (size_t) decBytes); + } + + decBufIndex = (decBufIndex + 1) % 2; + } +} + + +int compare(FILE* fp0, FILE* fp1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = read_bin(fp0, b0, sizeof(b0)); + const size_t r1 = read_bin(fp1, b1, sizeof(b1)); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char* argv[]) +{ + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + sprintf(inpFilename, "%s", argv[1]); + sprintf(lz4Filename, "%s.lz4s-%d", argv[1], BLOCK_BYTES); + sprintf(decFilename, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + printf("compress : %s -> %s\n", inpFilename, lz4Filename); + test_compress(outFp, inpFp); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + printf("decompress : %s -> %s\n", lz4Filename, decFilename); + test_decompress(outFp, inpFp); + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c new file mode 100644 index 0000000..e236a44 --- /dev/null +++ b/examples/blockStreaming_lineByLine.c @@ -0,0 +1,207 @@ +// LZ4 streaming API example : line-by-line logfile compression +// Copyright : Takayuki Matsuoka + + +#define _CRT_SECURE_NO_WARNINGS // for MSVC +#include "lz4.h" + +#include +#include +#include +#include + +static size_t write_uint16(FILE* fp, uint16_t i) +{ + return fwrite(&i, sizeof(i), 1, fp); +} + +static size_t write_bin(FILE* fp, const void* array, int arrayBytes) +{ + return fwrite(array, 1, arrayBytes, fp); +} + +static size_t read_uint16(FILE* fp, uint16_t* i) +{ + return fread(i, sizeof(*i), 1, fp); +} + +static size_t read_bin(FILE* fp, void* array, int arrayBytes) +{ + return fread(array, 1, arrayBytes, fp); +} + + +static void test_compress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + LZ4_stream_t* const lz4Stream = LZ4_createStream(); + char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); + char* const inpBuf = malloc(ringBufferBytes); + int inpOffset = 0; + + for ( ; ; ) + { + char* const inpPtr = &inpBuf[inpOffset]; + +#if 0 + // Read random length data to the ring buffer. + const int randomLength = (rand() % messageMaxBytes) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; +#else + // Read line to the ring buffer. + int inpBytes = 0; + if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) + break; + inpBytes = (int) strlen(inpPtr); +#endif + + { + const int cmpBytes = LZ4_compress_continue( + lz4Stream, inpPtr, cmpBuf, inpBytes); + if (cmpBytes <= 0) break; + write_uint16(outFp, (uint16_t) cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + // Add and wraparound the ringbuffer offset + inpOffset += inpBytes; + if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; + } + } + write_uint16(outFp, 0); + + free(inpBuf); + free(cmpBuf); + LZ4_freeStream(lz4Stream); +} + + +static void test_decompress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode(); + char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); + char* const decBuf = malloc(ringBufferBytes); + int decOffset = 0; + + for ( ; ; ) + { + uint16_t cmpBytes = 0; + + if (read_uint16(inpFp, &cmpBytes) != 1) break; + if (cmpBytes <= 0) break; + if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); + if (decBytes <= 0) break; + write_bin(outFp, decPtr, decBytes); + + // Add and wraparound the ringbuffer offset + decOffset += decBytes; + if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; + } + } + + free(decBuf); + free(cmpBuf); + LZ4_freeStreamDecode(lz4StreamDecode); +} + + +static int compare(FILE* f0, FILE* f1) +{ + int result = 0; + const size_t tempBufferBytes = 65536; + char* const b0 = malloc(tempBufferBytes); + char* const b1 = malloc(tempBufferBytes); + + while(0 == result) + { + const size_t r0 = fread(b0, 1, tempBufferBytes, f0); + const size_t r1 = fread(b1, 1, tempBufferBytes, f1); + + result = (int) r0 - (int) r1; + + if (0 == r0 || 0 == r1) break; + if (0 == result) result = memcmp(b0, b1, r0); + } + + free(b1); + free(b0); + return result; +} + + +int main(int argc, char* argv[]) +{ + enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + }; + + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if (argc < 2) + { + printf("Please specify input filename\n"); + return 0; + } + + sprintf(inpFilename, "%s", argv[1]); + sprintf(lz4Filename, "%s.lz4s", argv[1]); + sprintf(decFilename, "%s.lz4s.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if (0 == cmp) + printf("Verify : OK\n"); + else + printf("Verify : NG\n"); + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/examples/blockStreaming_ringBuffer.c b/examples/blockStreaming_ringBuffer.c new file mode 100644 index 0000000..725e375 --- /dev/null +++ b/examples/blockStreaming_ringBuffer.c @@ -0,0 +1,186 @@ +// LZ4 streaming API example : ring buffer +// Copyright : Takayuki Matsuoka + + +#define _CRT_SECURE_NO_WARNINGS // for MSVC +#include "lz4.h" + +#include +#include +#include +#include + +enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + DICT_BYTES = 65536, +}; + + +size_t write_int32(FILE* fp, int32_t i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, int arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int32(FILE* fp, int32_t* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, int arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + LZ4_stream_t lz4Stream_body = { 0 }; + LZ4_stream_t* lz4Stream = &lz4Stream_body; + + static char inpBuf[RING_BUFFER_BYTES]; + int inpOffset = 0; + + for(;;) { + // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. + char* const inpPtr = &inpBuf[inpOffset]; + const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; + + { + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + const int cmpBytes = LZ4_compress_continue(lz4Stream, inpPtr, cmpBuf, inpBytes); + if(cmpBytes <= 0) break; + write_int32(outFp, cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + inpOffset += inpBytes; + + // Wraparound the ringbuffer offset + if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0; + } + } + + write_int32(outFp, 0); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + static char decBuf[RING_BUFFER_BYTES]; + int decOffset = 0; + LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; + + for(;;) { + int cmpBytes = 0; + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + + { + const size_t r0 = read_int32(inpFp, &cmpBytes); + if(r0 != 1 || cmpBytes <= 0) break; + + const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); + if(r1 != (size_t) cmpBytes) break; + } + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); + if(decBytes <= 0) break; + decOffset += decBytes; + write_bin(outFp, decPtr, decBytes); + + // Wraparound the ringbuffer offset + if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; + } + } +} + + +int compare(FILE* f0, FILE* f1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = fread(b0, 1, sizeof(b0), f0); + const size_t r1 = fread(b1, 1, sizeof(b1), f1); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char** argv) +{ + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + sprintf(inpFilename, "%s", argv[1]); + sprintf(lz4Filename, "%s.lz4s-%d", argv[1], 0); + sprintf(decFilename, "%s.lz4s-%d.dec", argv[1], 0); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + test_compress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("Verify : OK\n"); + } else { + printf("Verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/examples/printVersion.c b/examples/printVersion.c index f86e469..8607139 100644 --- a/examples/printVersion.c +++ b/examples/printVersion.c @@ -1,8 +1,13 @@ +// LZ4 trivial example : print Library version number +// Copyright : Takayuki Matsuoka & Yann Collet + + #include #include "lz4.h" int main(int argc, char** argv) { - printf("Hello, LZ4 (version = %d)\n", LZ4_versionNumber()); + (void)argc; (void)argv; + printf("Hello World ! LZ4 Library version = %d\n", LZ4_versionNumber()); return 0; } -- cgit v0.12