diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2014-08-26 13:27:22 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2014-08-26 13:27:22 (GMT) |
commit | 43465ae8ff2ff07020ef6183172c4160ec6415fb (patch) | |
tree | defb02aa2031b29a67ba0f9bd0dd854e8d8be363 /examples/blockStreaming_lineByLine.c | |
parent | 18392cd5e186ae2d3420130dc0f5451b4d916e3e (diff) | |
download | lz4-43465ae8ff2ff07020ef6183172c4160ec6415fb.zip lz4-43465ae8ff2ff07020ef6183172c4160ec6415fb.tar.gz lz4-43465ae8ff2ff07020ef6183172c4160ec6415fb.tar.bz2 |
Added : examples from Takayuki Matsuoka
Diffstat (limited to 'examples/blockStreaming_lineByLine.c')
-rw-r--r-- | examples/blockStreaming_lineByLine.c | 207 |
1 files changed, 207 insertions, 0 deletions
diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c new file mode 100644 index 0000000..e236a44 --- /dev/null +++ b/examples/blockStreaming_lineByLine.c @@ -0,0 +1,207 @@ +// LZ4 streaming API example : line-by-line logfile compression +// Copyright : Takayuki Matsuoka + + +#define _CRT_SECURE_NO_WARNINGS // for MSVC +#include "lz4.h" + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +static size_t write_uint16(FILE* fp, uint16_t i) +{ + return fwrite(&i, sizeof(i), 1, fp); +} + +static size_t write_bin(FILE* fp, const void* array, int arrayBytes) +{ + return fwrite(array, 1, arrayBytes, fp); +} + +static size_t read_uint16(FILE* fp, uint16_t* i) +{ + return fread(i, sizeof(*i), 1, fp); +} + +static size_t read_bin(FILE* fp, void* array, int arrayBytes) +{ + return fread(array, 1, arrayBytes, fp); +} + + +static void test_compress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + LZ4_stream_t* const lz4Stream = LZ4_createStream(); + char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); + char* const inpBuf = malloc(ringBufferBytes); + int inpOffset = 0; + + for ( ; ; ) + { + char* const inpPtr = &inpBuf[inpOffset]; + +#if 0 + // Read random length data to the ring buffer. + const int randomLength = (rand() % messageMaxBytes) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; +#else + // Read line to the ring buffer. + int inpBytes = 0; + if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) + break; + inpBytes = (int) strlen(inpPtr); +#endif + + { + const int cmpBytes = LZ4_compress_continue( + lz4Stream, inpPtr, cmpBuf, inpBytes); + if (cmpBytes <= 0) break; + write_uint16(outFp, (uint16_t) cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + // Add and wraparound the ringbuffer offset + inpOffset += inpBytes; + if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; + } + } + write_uint16(outFp, 0); + + free(inpBuf); + free(cmpBuf); + LZ4_freeStream(lz4Stream); +} + + +static void test_decompress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode(); + char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); + char* const decBuf = malloc(ringBufferBytes); + int decOffset = 0; + + for ( ; ; ) + { + uint16_t cmpBytes = 0; + + if (read_uint16(inpFp, &cmpBytes) != 1) break; + if (cmpBytes <= 0) break; + if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); + if (decBytes <= 0) break; + write_bin(outFp, decPtr, decBytes); + + // Add and wraparound the ringbuffer offset + decOffset += decBytes; + if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; + } + } + + free(decBuf); + free(cmpBuf); + LZ4_freeStreamDecode(lz4StreamDecode); +} + + +static int compare(FILE* f0, FILE* f1) +{ + int result = 0; + const size_t tempBufferBytes = 65536; + char* const b0 = malloc(tempBufferBytes); + char* const b1 = malloc(tempBufferBytes); + + while(0 == result) + { + const size_t r0 = fread(b0, 1, tempBufferBytes, f0); + const size_t r1 = fread(b1, 1, tempBufferBytes, f1); + + result = (int) r0 - (int) r1; + + if (0 == r0 || 0 == r1) break; + if (0 == result) result = memcmp(b0, b1, r0); + } + + free(b1); + free(b0); + return result; +} + + +int main(int argc, char* argv[]) +{ + enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + }; + + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if (argc < 2) + { + printf("Please specify input filename\n"); + return 0; + } + + sprintf(inpFilename, "%s", argv[1]); + sprintf(lz4Filename, "%s.lz4s", argv[1]); + sprintf(decFilename, "%s.lz4s.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if (0 == cmp) + printf("Verify : OK\n"); + else + printf("Verify : NG\n"); + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} |