diff options
| -rwxr-xr-x | examples/HCStreaming_ringBuffer.c | 208 | ||||
| -rw-r--r-- | examples/Makefile | 8 | ||||
| -rwxr-xr-x | lz4hc.c | 70 |
3 files changed, 263 insertions, 23 deletions
diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c new file mode 100755 index 0000000..3532fa3 --- /dev/null +++ b/examples/HCStreaming_ringBuffer.c @@ -0,0 +1,208 @@ +// LZ4 HC streaming API example : ring buffer +// Based on previous work from Takayuki Matsuoka + + +/************************************** + Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS // for MSVC +# define snprintf sprintf_s +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + + +/************************************** + Includes +**************************************/ +#include "lz4hc.h" +#include "lz4.h" + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + DICT_BYTES = 65536, +}; + + +size_t write_int32(FILE* fp, int32_t i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, int arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int32(FILE* fp, int32_t* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, int arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + LZ4_streamHC_t lz4Stream_body = { 0 }; + LZ4_streamHC_t* lz4Stream = &lz4Stream_body; + + static char inpBuf[RING_BUFFER_BYTES]; + int inpOffset = 0; + + for(;;) + { + // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. + char* const inpPtr = &inpBuf[inpOffset]; + const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; + + { + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes); + if(cmpBytes <= 0) break; + write_int32(outFp, cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + inpOffset += inpBytes; + + // Wraparound the ringbuffer offset + if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) + inpOffset = 0; + } + } + + write_int32(outFp, 0); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + static char decBuf[RING_BUFFER_BYTES]; + int decOffset = 0; + LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; + + for(;;) { + int cmpBytes = 0; + char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + + { + const size_t r0 = read_int32(inpFp, &cmpBytes); + size_t r1; + if(r0 != 1 || cmpBytes <= 0) break; + + r1 = read_bin(inpFp, cmpBuf, cmpBytes); + if(r1 != (size_t) cmpBytes) break; + } + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); + if(decBytes <= 0) + break; + decOffset += decBytes; + write_bin(outFp, decPtr, decBytes); + + // Wraparound the ringbuffer offset + if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0; + } + } +} + + +int compare(FILE* f0, FILE* f1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = fread(b0, 1, sizeof(b0), f0); + const size_t r1 = fread(b1, 1, sizeof(b1), f1); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char** argv) +{ + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0); + snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + + test_compress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("Verify : OK\n"); + } else { + printf("Verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/examples/Makefile b/examples/Makefile index 5704c55..b5f9a7e 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -58,7 +58,7 @@ endif default: all -all: printVersion doubleBuffer ringBuffer lineCompress +all: printVersion doubleBuffer ringBuffer ringBufferHC lineCompress printVersion: $(LZ4DIR)/lz4.c printVersion.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -69,6 +69,9 @@ doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c $(CC) $(FLAGS) $^ -o $@$(EXT) +ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -76,10 +79,11 @@ test : all ./printVersion$(EXT) ./doubleBuffer$(EXT) $(TESTFILE) ./ringBuffer$(EXT) $(TESTFILE) + ./ringBufferHC$(EXT) $(TESTFILE) ./lineCompress$(EXT) $(TESTFILE) clean: @rm -f core *.o *.dec *-0 *-8192 *.lz4s \ - printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT) + printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT) @echo Cleaning completed @@ -379,6 +379,21 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) } +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content + // Note : need to handle risk of index overflow + // Use only one memory segment for dict, so any previous External Dict is lost at this stage + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block +} + + static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit) { const BYTE* const p1Start = p1; @@ -397,21 +412,6 @@ static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* co } -static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + 4) - LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content - // Note : need to handle risk of index overflow - // Use only one memory segment for dict, so any previous External Dict is lost at this stage - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block -} - - FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, @@ -618,8 +618,6 @@ static int LZ4HC_compress_generic ( if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL; if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL; maxNbAttempts = 1 << compressionLevel; - /* check if blocks follow each other */ - if (ip != ctx->end) LZ4HC_setExternalDict(ctx, ip); ctx->end += inputSize; ip++; @@ -866,16 +864,46 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int /* compression */ +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize, limitedOutput_directive limit) +{ + /* auto-init if forgotten */ + if (dsPtr->base == NULL) + LZ4HC_init (dsPtr, (const BYTE*) source); + + /* check if blocks follow each other */ + if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit; + const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit; + if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd)) + { + dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase); + if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit; + } + } + + return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit); +} + int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) { - if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); + //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) + // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit); + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); } int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { - if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); - return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); + //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) + // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source); + //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput); + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); } |